alternate data processing

package.json

@@ -33,7 +33,6 @@
     "build": "npx tsc",
     "startServer": "node out/serverMain.js",
     "startClient": "node out/clientMain.js",
-    "startProcess": "node out/processMain.js",
     "documentation": "redocly build-docs .\\docs\\v1.yml --output=docs\\index.html",
     "test": "jest --runInBand"
   }

src/cache/cacheUtils.ts

@@ -1,97 +0,0 @@
-import cache from './redis'
-/**
- * Shows the data for one weapon
- * @param weapon
- * @param server
- * @param player
- * @returns
- */
-export async function getWeaponReport(
-  weapon: string,
-  server?: number,
-  player?: string
-) {
-  const serverPrefix = !isNaN(Number(server?.toString()))
-    ? `servers:${server}:`
-    : ''
-  const playerPrefix = player ? `players:${player}:` : ''
-  const cacheLocation = serverPrefix + playerPrefix + `weapons:` + weapon
-  const data = await cache.HGETALL(cacheLocation)
-  delete data.last_entry
-  return data
-}
-
-export function getWeaponSet(server?: number, player?: string) {
-  const serverPrefix = !isNaN(Number(server?.toString()))
-    ? `servers:${server}:`
-    : ''
-  const playerPrefix = player ? `players:${player}:` : ''
-  const cacheLocation = serverPrefix + playerPrefix + `weapons`
-  return cache.SMEMBERS(cacheLocation)
-}
-
-/**
- * Show the processed list of weapons
- * @param server
- * @param player
- * @returns string
- */
-export async function getWeaponList(server?: string, player?: string) {
-  const serverPrefix = !isNaN(Number(server?.toString()))
-    ? `servers:${server}:`
-    : ''
-  const playerPrefix = player ? `players:${player}:` : ''
-  return JSON.parse(
-    (await cache.GET(serverPrefix + playerPrefix + `weapons:processedList`)) ||
-      '{}'
-  )
-}
-
-/**
- * Show processed data for a player
- * @param player
- * @param server
- * @param weapon
- * @returns
- */
-export async function getPlayerReport(
-  player: string,
-  server?: number,
-  weapon?: string
-) {
-  const serverPrefix = !isNaN(Number(server?.toString()))
-    ? `servers:${server}:`
-    : ''
-  const weaponPrefix = weapon ? `weapons:${weapon}:` : ''
-  const cacheLocation = serverPrefix + weaponPrefix + `players:` + player
-  const data = await cache.HGETALL(cacheLocation)
-  delete data.last_entry
-  return data
-}
-
-/**
- * Shows processed list of players
- * @param server
- * @param weapon
- * @returns string
- */
-export async function getPlayerList(server?: number, weapon?: string) {
-  const serverPrefix = !isNaN(Number(server?.toString()))
-    ? `servers:${server}:`
-    : ''
-  const weaponPrefix = weapon ? `weapons:${weapon}:` : ''
-
-  return JSON.parse(
-    (await cache.GET(serverPrefix + weaponPrefix + `players:processedList`)) ||
-      '{}'
-  )
-}
-
-export function getPlayerSet(server?: number, weapon?: string) {
-  const serverPrefix = !isNaN(Number(server?.toString()))
-    ? `servers:${server}:`
-    : ''
-  const weaponPrefix = weapon ? `weapons:${weapon}:` : ''
-  const cacheLocation = serverPrefix + weaponPrefix + `players`
-  return cache.SMEMBERS(cacheLocation)
-}

src/cache/redis.ts

@@ -1,21 +0,0 @@
-import { createClient } from 'redis'
-
-import * as dotenv from 'dotenv'
-dotenv.config()
-
-const client = createClient({
-  url: process.env.REDIS_URL,
-  username: process.env.REDIS_USER,
-  password: process.env.REDIS_PASSWORD
-})
-
-client.on('error', (err) => console.log('Redis Client Error', err))
-
-export function cacheReady() {
-  return client.connect()
-}
-export default client
-
-//await client.set('key', 'value');
-//const value = await client.get('key');
-//await client.disconnect();

src/client/client.ts

@@ -1,86 +1,81 @@
 import { Router } from 'express'
 import { param, query } from 'express-validator'
 import { validateErrors } from '../common'
-import db from '../db/db'
+import { allData } from '../process/process'
-import client from '../cache/redis'
+
 const router = Router()
 //timeout middleware ?
 router.get('/*', (req, res, next) => {
   next()
 })
 
-router.get(
+function filters(e: any, query: any) {
-  '/weapons/:weaponId',
+  return (
-  param(['weaponId']).exists().isString(),
+    (query.player ? query.player == e.attacker_id : true) &&
-  query(['player', 'server']).optional().toInt().isInt(),
+    (query.server ? query.server == e.servername : true) &&
-  validateErrors,
+    (query.host ? Number(query.host) == e.host : true) &&
-  async (req, res) => {
+    (query.map ? query.map == e.map : true) &&
-    let path = ''
+    (query.weapon ? query.weapon == e.cause_of_death : true) &&
-    if (req.query.server) path = path + `servers.${req.query.server}.`
+    (query.gamemode ? query.gamemode == e.game_mode : true)
-    if (req.query.player) path = path + `players.${req.query.player}.`
+  )
-    path = path + `weapons.${req.params.weaponId}`
+}
-    if (await client.json.type('kills', path)) return res.status(200).send(await client.json.get('kills', { path }))
-    res.status(404).send()
-  }
-)
 
 router.get(
-  '/weapons/',
+  '/:dataType',
-  query(['player', 'server']).optional().toInt().isInt(),
+  param('dataType')
+    .custom(
+      (e) => e == 'weapons' || e == 'players' || e == 'maps' || e == 'servers'
+    )
+    .withMessage('Only weapons, players, maps or servers are valid paths'),
+  query(['player', 'host']).optional().toInt().isInt(),
+  query(['server', 'map', 'weapon', 'gamemode']).optional().isString(),
   validateErrors,
-  async (req, res) => {
+  (req, res) => {
-    let path = ''
+    const data: {
-    if (req.query.server) path = path + `servers.${req.query.server}.`
+      [key: string]: {
-    if (req.query.player) path = path + `players.${req.query.player}.`
+        deaths: number
-    path = path + `weapons`
+        kills: number
-    if (await client.json.type('kills', path)) {
+        max_distance: number
-      const data = await client.json.get('kills', { path: `weapons` }) as { [key: number]: any }
+        total_distance: number
-      return res.status(200).send(Object.fromEntries(Object.entries(data).map(([key, value]) => { return [key, { total_distance: value.total_distance, max_distance: value.max_distance, kills: value.kills, deaths: value.deaths }] })))
       }
-    res.status(404).send()
+    } = {}
+    let index: 'cause_of_death' | 'attacker_id' | 'map' | 'servername'
+    switch (req.params.dataType) {
+      case 'weapons':
+        index = 'cause_of_death'
+        break
+      case 'players':
+        index = 'attacker_id'
+        break
+      case 'maps':
+        index = 'map'
+        break
+      case 'servers':
+        index = 'servername'
+        break
+      default:
+        return res.status(400).send()
+    }
+    allData
+      .filter((e) => filters(e, req.query))
+      .forEach((e) => {
+        if (!data[e[index]])
+          data[e[index]] = {
+            deaths: 0,
+            kills: 0,
+            max_distance: 0,
+            total_distance: 0
+          }
+        data[e[index]].deaths += Number(e.deaths)
+        data[e[index]].kills += Number(e.kills)
+        data[e[index]].total_distance += Number(e.total_distance)
+        data[e[index]].max_distance = Math.max(
+          data[e[index]].max_distance,
+          Number(e.max_distance)
+        )
+      })
+    res.status(200).send(data)
   }
 )
 
-router.get(
-  '/players/:playerId',
-  param(['playerId']).exists().toInt().isInt(),
-  query(['server']).optional().toInt().isInt(),
-  query('weapon').optional().isString(),
-  validateErrors,
-  async (req, res) => {
-    let path = ''
-    if (req.query.server) path = path + `servers.${req.query.server}.`
-    if (req.query.weapon) path = path + `weapons.${req.query.weapon}.`
-    path = path + `players.${req.params.playerId}`
-    if (await client.json.type('kills', path)) return res.status(200).send(await client.json.get('kills', { path }))
-    res.status(404).send()
-  }
-)
-
-router.get(
-  '/players/',
-  query(['server']).optional().toInt().isInt(),
-  query('weapon').optional().isString(),
-  validateErrors,
-  async (req, res) => {
-    let path = ''
-    if (req.query.server) path = path + `servers.${req.query.server}.`
-    if (req.query.weapon) path = path + `weapons.${req.query.weapon}.`
-    path = path + `players`
-    if (await client.json.type('kills', path)) {
-      const data = await client.json.get('kills', { path: `players` }) as { [key: number]: any }
-      return res.status(200).send(Object.fromEntries(Object.entries(data).map(([key, value]) => { return [key, { total_distance: value.total_distance, max_distance: value.max_distance, kills: value.kills, deaths: value.deaths }] })))
-    }
-    res.status(404).send()
-  }
-)
-
-//router.get('/maps/', (req, res, next) => {})
-
-router.get('/servers/', async (req, res) => {
-  const data = await client.json.get('kills', { path: `servers` }) as { [key: number]: any }
-  return res.status(200).send(Object.fromEntries(Object.entries(data).map(([key, value]) => { return [key, value.data] })))
-})
-//router.get('/servers/:serverId/', (req, res, next) => {})
-
 export default router

src/clientMain.ts

@@ -3,8 +3,8 @@ dotenv.config()
 import express from 'express'
 import cors from 'cors'
 import client from './client/client'
-import db, { dbReady } from './db/db'
+import { dbReady } from './db/db'
-import cache, { cacheReady } from './cache/redis'
+import processAll from './process/process'
 
 const app = express()
 const port = 3000
@@ -19,13 +19,11 @@ app.get('/', (req, res) => {
 app.use('/', client)
 
 export default
-  new Promise((resolve, reject) => {
+  new Promise(async (resolve, reject) => {
-    dbReady().then((e) => {
+    await dbReady()
-      cacheReady().then((e) => {
+    await processAll()
     const listenServer = app.listen(port, '0.0.0.0', () => {
       console.log(`Tone client api listening on port ${port}`)
       resolve(listenServer)
     })
   })
-    })
-  })

src/db/migrations/2023-04-09-02 server auth patch.ts

@@ -11,20 +11,26 @@ const pgClient = new Client({
 export async function up(db: Kysely<any>): Promise<void> {
   await pgClient.connect()
   await pgClient.query('ALTER TABLE kill ADD servername character varying NULL;')
+  await pgClient.query('ALTER TABLE kill ADD host integer NULL;')
   await pgClient.query('UPDATE kill SET servername = server.name FROM server WHERE server.id = kill.server;')
   await pgClient.query('ALTER TABLE kill ALTER COLUMN servername SET NOT NULL;')
   await pgClient.query('ALTER TABLE kill DROP COLUMN server;')
 
-  await pgClient.query('ALTER TABLE server RENAME TO hoster')
+  //await pgClient.query('ALTER TABLE server RENAME TO hoster')
-  await pgClient.query('ALTER TABLE hoster DROP COLUMN description')
+  //await pgClient.query('ALTER TABLE hoster DROP COLUMN description')
-  await pgClient.query('ALTER TABLE kill ADD COLUMN host integer NULL')
+  //await pgClient.query('ALTER TABLE kill ADD COLUMN host integer NULL')
+  await pgClient.query('CREATE TABLE host (name character varying, token character varying)')
+
   //Update the hosts column once the hoster table is manually updated
+  await pgClient.end()
 }
 
 export async function down(db: Kysely<any>): Promise<void> {
-  await pgClient.query('ALTER TABLE kill DROP COLUMN host')
+  await pgClient.connect()
-  await pgClient.query('ALTER TABLE hoster add description character varying NULL')
+  //await pgClient.query('ALTER TABLE kill DROP COLUMN host')
-  await pgClient.query('ALTER TABLE hoster RENAME TO server')
+  //await pgClient.query('ALTER TABLE hoster add description character varying NULL')
+  //await pgClient.query('ALTER TABLE hoster RENAME TO server')
+  await pgClient.query('DROP TABLE host')
 
   await pgClient.query(`ALTER TABLE kill ADD server integer NULL;`)
   await pgClient.query('UPDATE kill SET server = server.id FROM server where server.name = kill.servername;')

src/db/migrations/2023-04-19-01 last aggregate.ts

@@ -0,0 +1,46 @@
+import { Kysely, sql } from 'kysely'
+import { Client } from 'pg'
+
+const pgClient = new Client({
+    host: process.env.POSTGRES_HOST,
+    database: process.env.POSTGRES_DATABASE,
+    user: process.env.POSTGRES_USER,
+    password: process.env.POSTGRES_PASSWORD
+})
+
+export async function up(db: Kysely<any>): Promise<void> {
+    await pgClient.connect()
+    await pgClient.query(`CREATE OR REPLACE FUNCTION public.first_agg (anyelement, anyelement)
+  RETURNS anyelement
+  LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE AS
+'SELECT $1';`)
+
+    await pgClient.query(`CREATE AGGREGATE public.first (anyelement) (
+    SFUNC    = public.first_agg
+  , STYPE    = anyelement
+  , PARALLEL = safe
+  );`)
+
+    await pgClient.query(`CREATE OR REPLACE FUNCTION public.last_agg (anyelement, anyelement)
+  RETURNS anyelement
+  LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE AS
+'SELECT $2';`)
+
+
+    await pgClient.query(`CREATE AGGREGATE public.last (anyelement) (
+    SFUNC    = public.last_agg
+  , STYPE    = anyelement
+  , PARALLEL = safe
+  );`)
+    //Update the hosts column once the hoster table is manually updated
+    await pgClient.end()
+}
+
+export async function down(db: Kysely<any>): Promise<void> {
+    await pgClient.connect()
+    pgClient.query('DROP AGGREGATE public.last')
+    pgClient.query('DROP FUNCTION public.last_agg')
+    pgClient.query('DROP AGGREGATE public.first')
+    pgClient.query('DROP AGGREGATE public.first_agg')
+    await pgClient.end()
+}

src/process/onKill.ts

@@ -1,6 +1,5 @@
 import { Client } from 'pg'
-import client from '../cache/redis'
+import { allData } from './process'
-import { genPrefix } from './process'
 
 const pgClient = new Client({
     host: process.env.POSTGRES_HOST,
@@ -13,98 +12,63 @@ export default async function listenKills() {
     await pgClient.connect()
     await pgClient.query('LISTEN new_kill')
 
-    pgClient.on('notification', async (data) => {
+    pgClient.on('notification', (data) => {
         if (!data.payload) return
         const payload = JSON.parse(data.payload)
-        await updateGlobal(payload)
+        let killEntry = allData.find(
+            (e) =>
+                e.attacker_id == payload.attacker_id &&
+                e.cause_of_death == payload.cause_of_death &&
+                e.game_mode == payload.game_mode &&
+                e.host == payload.host &&
+                e.map == payload.map &&
+                e.servername == payload.servername
+        )
+        let deathEntry = allData.find(
+            (e) =>
+                e.attacker_id == payload.victim_id &&
+                e.cause_of_death == payload.cause_of_death &&
+                e.game_mode == payload.game_mode &&
+                e.host == payload.host &&
+                e.map == payload.map &&
+                e.servername == payload.servername
+        )
+        if (!killEntry) {
+            killEntry = {
+                kills: 0,
+                deaths: 0,
+                max_distance: 0,
+                total_distance: 0,
+                attacker_id: payload.attacker_id,
+                cause_of_death: payload.cause_of_death,
+                game_mode: payload.game_mode,
+                host: payload.host,
+                map: payload.map,
+                servername: payload.servername
+            }
+            allData.push(killEntry)
+        }
+
+        if (!deathEntry) {
+            deathEntry = {
+                kills: 0,
+                deaths: 0,
+                max_distance: 0,
+                total_distance: 0,
+                attacker_id: payload.victim_id,
+                cause_of_death: payload.victim_current_weapon,
+                game_mode: payload.game_mode,
+                host: payload.host,
+                map: payload.map,
+                servername: payload.servername
+            }
+            allData.push(deathEntry)
+        }
+
+        killEntry.kills++
+        killEntry.total_distance += payload.distance
+        killEntry.max_distance = Math.max(killEntry.max_distance, payload.distance)
+        deathEntry.deaths++
     })
     return pgClient
 }
-
-
-async function updateGlobal({ cause_of_death, distance, attacker_id, victim_id, servername, host }: { cause_of_death: string, distance: number, attacker_id: string, victim_id: string, servername: string, host: number }) {
-    servername = servername.replace(/[^a-z0-9]/gi, '')
-    const promises: Promise<any>[] = []
-
-    if (!await client.json.type('kills')) await client.json.set('kills', '', { data: {}, weapons: {}, servers: {}, players: {} })
-
-    if (!await client.json.type('kills', genPrefix.weapon({ cause_of_death }))) await client.json.set('kills', genPrefix.weapon({ cause_of_death }), { players: {}, max_distance: 0, kills: 0, total_distance: 0 })
-    if (!await client.json.type('kills', genPrefix.player({ attacker_id }))) await client.json.set('kills', genPrefix.player({ attacker_id }), { weapons: {}, max_distance: 0, kills: 0, deaths: 0, total_distance: 0 })
-    if (!await client.json.type('kills', `servers.${host}`)) await client.json.set('kills', `servers.${host}`, {})
-    if (!await client.json.type('kills', `servers.${host}.${servername}`)) await client.json.set('kills', `servers.${host}.${servername}`, { data: {}, weapons: {}, players: {}, max_distance: 0, kills: 0, total_distance: 0 })
-    if (!await client.json.type('kills', genPrefix.weapon({ cause_of_death, server: `${host}.${servername}` }))) await client.json.set('kills', genPrefix.weapon({ cause_of_death, server: `${host}.${servername}` }), { players: {}, max_distance: 0, kills: 0, total_distance: 0 })
-    if (!await client.json.type('kills', genPrefix.player({ attacker_id, server: `${host}.${servername}` }))) await client.json.set('kills', genPrefix.player({ attacker_id, server: `${host}.${servername}` }), { weapons: {}, max_distance: 0, kills: 0, deaths: 0, total_distance: 0 })
-
-    if (!await client.json.type('kills', genPrefix.player({ attacker_id: victim_id, server: `${host}.${servername}` }))) await client.json.set('kills', genPrefix.player({ attacker_id: victim_id, server: `${host}.${servername}` }), { weapons: {}, max_distance: 0, kills: 0, deaths: 0, total_distance: 0 })
-    if (!await client.json.type('kills', genPrefix.player({ attacker_id: victim_id }))) await client.json.set('kills', genPrefix.player({ attacker_id: victim_id }), { weapons: {}, max_distance: 0, kills: 0, deaths: 0, total_distance: 0 })
-
-    if (!await client.json.type('kills', genPrefix.weaponPlayers({ attacker_id: victim_id, cause_of_death }))) await client.json.set('kills', genPrefix.weaponPlayers({ attacker_id: victim_id, cause_of_death }), { max_distance: 0, kills: 0, deaths: 0, total_distance: 0 })
-    if (!await client.json.type('kills', genPrefix.playerWeapons({ attacker_id: victim_id, cause_of_death }))) await client.json.set('kills', genPrefix.playerWeapons({ attacker_id: victim_id, cause_of_death }), { max_distance: 0, kills: 0, total_distance: 0 })
-
-    if (!await client.json.type('kills', genPrefix.weaponPlayers({ attacker_id: victim_id, cause_of_death, server: `${host}.${servername}` }))) await client.json.set('kills', genPrefix.weaponPlayers({ attacker_id: victim_id, cause_of_death, server: `${host}.${servername}` }), { max_distance: 0, kills: 0, deaths: 0, total_distance: 0 })
-    if (!await client.json.type('kills', genPrefix.playerWeapons({ attacker_id: victim_id, cause_of_death, server: `${host}.${servername}` }))) await client.json.set('kills', genPrefix.playerWeapons({ attacker_id: victim_id, cause_of_death, server: `${host}.${servername}` }), { max_distance: 0, kills: 0, total_distance: 0 })
-
-    promises.push(updatePath(`data`, { distance }))
-    //servers.1
-    promises.push(updatePath(`servers.${host}.${servername}`, { distance }).then(() => Promise.all([
-        //servers.1.players.123456789
-        updatePath(`servers.${host}.${servername}.players.${attacker_id}`, { distance }).then(async () => {
-            if (!await client.json.type('kills', `servers.${host}.${servername}.players.${victim_id}.deaths`)) await client.json.set('kills', `servers.${host}.${servername}.players.${victim_id}.deaths`, 0)
-            await client.json.numIncrBy('kills', `servers.${host}.${servername}.players.${victim_id}.deaths`, 1)
-            //servers.1.players.123456789.weapons.epg
-            await updatePath(`servers.${host}.${servername}.players.${attacker_id}.weapons.${cause_of_death}`, { distance })
-        }
-
-        ),
-        //servers.1.weapons.epg
-        updatePath(`servers.${host}.${servername}.weapons.${cause_of_death}`, { distance }).then(async () => {
-            //servers.1.weapons.epg.players.123456789
-            await updatePath(`servers.${host}.${servername}.weapons.${cause_of_death}.players.${attacker_id}`, { distance })
-            if (!await client.json.type('kills', `servers.${host}.${servername}.players.${victim_id}.deaths`)) await client.json.set('kills', `servers.${host}.${servername}.weapons.${cause_of_death}.players.${victim_id}`, 0)
-            await client.json.numIncrBy('kills', `servers.${host}.${servername}.weapons.${cause_of_death}.players.${victim_id}.deaths`, 1)
-        }
-
-        )])
-    ))
-
-
-    //players.123456789
-    promises.push(updatePath(`players.${attacker_id}`, { distance }).then(async () => {
-        if (!await client.json.type('kills', `players.${victim_id}.deaths`)) await client.json.set('kills', `players.${victim_id}.deaths`, 0)
-        await client.json.numIncrBy('kills', `players.${victim_id}.deaths`, 1)
-        //players.123456789.weapons.epg
-        await updatePath(`players.${attacker_id}.weapons.${cause_of_death}`, { distance })
-    }))
-
-
-    //weapons.epg
-    promises.push(updatePath(`weapons.${cause_of_death}`, { distance }).then(async () => {
-        //weapons.epg.players.123456789
-        await updatePath(`weapons.${cause_of_death}.players.${attacker_id}`, { distance })
-        if (!await client.json.type('kills', `weapons.${cause_of_death}.players.${victim_id}.deaths`)) await client.json.set('kills', `weapons.${cause_of_death}.players.${victim_id}.deaths`, 0)
-        await client.json.numIncrBy('kills', `weapons.${cause_of_death}.players.${victim_id}.deaths`, 1)
-    }
-    ))
-
-
-    await Promise.all(promises)
-}
-
-async function updatePath(path: string, { distance }: { distance: number }) {
-    await setupPath(path)
-    console.log(path + '.max_distance', await client.json.type('kills', path + '.max_distance'))
-    console.log(path, await client.json.type('kills', path))
-    if (!await client.json.type('kills', path)) await client.json.set('kills', path, {})
-    let transaction = client.multi()
-    transaction = transaction.json.numIncrBy('kills', `${path}.total_distance`, distance)
-    let max_distance = Number(await client.json.get('kills', { path: `${path}.max_distance` }))
-    if (isNaN(max_distance) || max_distance < distance) transaction = transaction.json.set('kills', `${path}.max_distance`, distance)
-    transaction = transaction.json.numIncrBy('kills', `${path}.kills`, 1)
-    await transaction.exec()
-}
-
-async function setupPath(path: string) {
-    if (!await client.json.type('kills', path + '.kills')) await client.json.set('kills', path + '.kills', 0)
-    if (!await client.json.type('kills', path + '.total_distance')) await client.json.set('kills', path + '.total_distance', 0)
-    if (!await client.json.type('kills', path + '.max_distance')) await client.json.set('kills', path + '.max_distance', 0)
-}

src/process/process.ts

@@ -1,35 +1,19 @@
 import db from '../db/db'
-const { count, max, sum } = db.fn
+const { count, max, sum, coalesce } = db.fn
-import client from '../cache/redis'
+import { sql } from 'kysely'
 
-export const genPrefix = {
+export let allData: Awaited<ReturnType<typeof processGlobalStats>>
-  global: ({ server }: { server?: string }) => {
-    return (server ? `servers.${server}.` : '') + 'data'
-  },
-  weapon: ({ cause_of_death, server }: { cause_of_death: string, server?: string }) => {
-    return (server ? `servers.${server}.` : '') + `weapons.${cause_of_death}`
-  },
-  player: ({ attacker_id, server }: { attacker_id: string, server?: string }) => {
-    return (server ? `servers.${server}.` : '') + `players.${attacker_id}`
-  },
-  playerWeapons: ({ cause_of_death, attacker_id, server }: { attacker_id: string, cause_of_death: string, server?: string }) => {
-    return (server ? `servers.${server}.` : '') + `players.${attacker_id}.weapons.${cause_of_death}`
-  },
-  weaponPlayers: ({ cause_of_death, attacker_id, server }: { attacker_id: string, cause_of_death: string, server?: string }) => {
-    return (server ? `servers.${server}.` : '') + `weapons.${cause_of_death}.players.${attacker_id}`
-  },
-}
 
 /**
- * Starts the process of populating the REDIS database globally
+ * populates allData
  * @returns
  */
 async function processAll() {
   console.log('Starting data calculation...')
   const timeStart = new Date()
 
-  if (!await client.json.type('kills')) await client.json.set('kills', '$', { data: {}, weapons: {}, players: {}, servers: {} })
+  allData = await processGlobalStats()
-  await Promise.all([processGlobalStats(), processServerStats()])
+
   console.log(
     'Data calculation finished. Took + ' +
     Math.abs(new Date().getTime() - timeStart.getTime()) / 1000 +
@@ -38,158 +22,71 @@ async function processAll() {
   if (process.env.ENVIRONMENT == 'production') {
     return
   }
-  setTimeout(processAll, 3600000)
 }
 
 async function processGlobalStats() {
-  let promises: Promise<any>[] = []
+  return await db
-  //Global kills
+    .with('kills', (db) =>
-  await db.selectFrom('kill').select([count('id').as('kills'), max('distance').as('max_distance'), sum('distance').as('total_distance')]).execute()
+      db
-    .then(async (data) => {
+        .selectFrom('kill')
-      let transaction = client.multi()
+        .select([
-      await Promise.all(data.map(async ({ kills, max_distance, total_distance }) => {
+          count<number>('id').as('kills'),
-        await processData(genPrefix.global({}), { total_distance, max_distance, kills }, transaction)
+          coalesce(
-      }))
+            max<number | null, 'distance'>('distance'),
-      return transaction.exec()
+            sql<number>`0`
-    })
+          ).as('max_distance'),
-
+          coalesce(sum<number | null>('distance'), sql<number>`0`).as(
-  //Global weapon kill
+            'total_distance'
-  await db.selectFrom('kill').select([count('id').as('kills'), max('distance').as('max_distance'), sum('distance').as('total_distance'), 'cause_of_death']).groupBy('cause_of_death').execute()
+          ),
-    .then(async (data) => {
+          'attacker_id',
-      let transaction = client.multi()
+          'cause_of_death',
-      await Promise.all(data.map(async ({ kills, max_distance, total_distance, cause_of_death }) => {
+          'map',
-        const prefix = genPrefix.weapon({ cause_of_death })
+          'game_mode',
-        if (!await client.json.type('kills', prefix)) await client.json.set('kills', prefix, { players: {} })
+          'servername',
-        await processData(prefix, { kills, max_distance, total_distance }, transaction)
+          'host'
-      }))
+        ])
-      return transaction.exec()
+        .whereRef('attacker_id', '!=', 'victim_id')
-    })
+        .groupBy([
-
+          'attacker_id',
-  //Global player kill
+          'cause_of_death',
-  await db.selectFrom('kill').select([count('id').as('kills'), max('distance').as('max_distance'), sum('distance').as('total_distance'), 'attacker_id']).groupBy('attacker_id').whereRef("attacker_id", '!=', 'victim_id').execute()
+          'map',
-    .then(async (data) => {
+          'servername',
-      let transaction = client.multi()
+          'host',
-      await Promise.all(data.map(async ({ kills, max_distance, total_distance, attacker_id }) => {
+          'game_mode'
-        const prefix = genPrefix.player({ attacker_id })
+        ])
-        if (!await client.json.type('kills', prefix)) await client.json.set('kills', prefix, { weapons: {} })
+    )
-        await processData(prefix, { kills, max_distance, total_distance }, transaction)
+    .with('deaths', (db) =>
-      }))
+      db
-      return transaction.exec()
+        .selectFrom('kill')
-    })
+        .select([
-  //Global player deaths
+          count<number>('id').as('deaths'),
-  await db.selectFrom('kill').select([count('id').as('kills'), 'victim_id']).groupBy('victim_id').execute()
+          'victim_id',
-    .then(async (data) => {
+          'victim_current_weapon',
-      let transaction = client.multi()
+          'map',
-      await Promise.all(data.map(async ({ kills, victim_id }) => {
+          'game_mode',
-        const prefix = genPrefix.player({ attacker_id: victim_id })
+          'servername',
-        transaction.json.set('kills', prefix + ".deaths", Number(kills))
+          'host'
-      }))
+        ])
-      return transaction.exec()
+        .groupBy([
-    })
+          'victim_id',
-
+          'victim_current_weapon',
-  //Player weapon kills
+          'map',
-  await db.selectFrom('kill').select([count('id').as('kills'), max('distance').as('max_distance'), sum('distance').as('total_distance'), 'attacker_id', 'cause_of_death']).groupBy(['attacker_id', 'cause_of_death']).whereRef("attacker_id", '!=', 'victim_id').execute()
+          'servername',
-    .then(async (data) => {
+          'host',
-      let transaction = client.multi()
+          'game_mode'
-      await Promise.all(data.map(async ({ kills, max_distance, total_distance, attacker_id, cause_of_death }) => {
+        ])
-        await processData(genPrefix.playerWeapons({ attacker_id, cause_of_death }), { kills, max_distance, total_distance }, transaction)
+    )
-        await processData(genPrefix.weaponPlayers({ attacker_id, cause_of_death }), { kills, max_distance, total_distance }, transaction)
+    .selectFrom('kills')
-      }))
+    .leftJoin('deaths', (join) =>
-      return transaction.exec()
+      join
-    })
+        .onRef('deaths.victim_id', '=', 'kills.attacker_id')
-  //player weapon deaths
+        .onRef('deaths.victim_current_weapon', '=', 'kills.cause_of_death')
-  await db.selectFrom('kill').select([count('id').as('kills'), 'victim_id', 'cause_of_death']).groupBy(['victim_id', 'cause_of_death']).execute()
+        .onRef('deaths.servername', '=', 'kills.servername')
-    .then(async (data) => {
+        .onRef('deaths.host', '=', 'kills.host')
-      let transaction = client.multi()
+        .onRef('deaths.game_mode', '=', 'kills.game_mode')
-      await Promise.all(data.map(async ({ kills, victim_id, cause_of_death }) => {
+    )
-        transaction.json.set('kills', genPrefix.weaponPlayers({ attacker_id: victim_id, cause_of_death }) + ".deaths", Number(kills))
+    .selectAll('kills')
-      }))
+    .select(sql<number>`COALESCE(deaths.deaths, 0)`.as('deaths'))
-      return transaction.exec()
+    .execute()
-    })
 }
-
-async function processServerStats() {
-  let promises: Promise<any>[] = []
-  //Server global kills
-  await db.selectFrom('kill')
-    .select([count('id').as('kills'), max('distance').as('max_distance'), sum('distance').as('total_distance'), 'servername', 'host'])
-    .groupBy(['servername', 'host']).execute().then(async (data) => {
-      let transaction = client.multi()
-      const p = data.map(async ({ kills, max_distance, total_distance, servername, host }) => {
-        servername = servername.replace(/[^a-z0-9]/gi, '')
-        if (!await client.json.type('kills', `servers.${host}`)) await client.json.set('kills', `servers.${host}`, {})
-        if (!await client.json.type('kills', `servers.${host}.${servername}`)) await client.json.set('kills', `servers.${host}.${servername}`, { data: {}, weapons: {}, players: {} })
-        await processData(genPrefix.global({ server: `${host}.${servername}` }), { total_distance, max_distance, kills }, transaction)
-      })
-      await Promise.all(p)
-      return transaction.exec()
-    })
-
-  //Server weapon kills
-  await db.selectFrom('kill')
-    .select([count('id').as('kills'), max('distance').as('max_distance'), sum('distance').as('total_distance'), 'cause_of_death', 'servername', 'host']).groupBy(['cause_of_death', 'servername', 'host']).execute().then(async (data) => {
-      let transaction = client.multi()
-      await Promise.all(data.map(async ({ kills, max_distance, total_distance, cause_of_death, servername, host }) => {
-        servername = servername.replace(/[^a-z0-9]/gi, '')
-        const prefix = genPrefix.weapon({ cause_of_death, server: `${host}.${servername}` })
-        if (!await client.json.type('kills', prefix)) await client.json.set('kills', prefix, { players: {} })
-        await processData(prefix, { kills, max_distance, total_distance }, transaction)
-      }))
-      return transaction.exec()
-    })
-  //server player kill
-  await db.selectFrom('kill').select([count('id').as('kills'), max('distance').as('max_distance'), sum('distance').as('total_distance'), 'attacker_id', 'servername', 'host']).groupBy(['attacker_id', 'servername', 'host']).whereRef("attacker_id", '!=', 'victim_id').execute()
-    .then(async (data) => {
-      let transaction = client.multi()
-      await Promise.all(data.map(async ({ kills, max_distance, total_distance, attacker_id, servername, host }) => {
-        servername = servername.replace(/[^a-z0-9]/gi, '')
-        const prefix = genPrefix.player({ attacker_id, server: `${host}.${servername}` })
-        if (!await client.json.type('kills', prefix)) await client.json.set('kills', prefix, { weapons: {} })
-        await processData(prefix, { kills, max_distance, total_distance }, transaction)
-      }))
-      return transaction.exec()
-    })
-  //server player deaths
-  await db.selectFrom('kill').select([count('id').as('kills'), 'victim_id', 'servername', 'host']).groupBy(['victim_id', 'servername', 'host']).execute()
-    .then(async (data) => {
-      let transaction = client.multi()
-      await Promise.all(data.map(async ({ kills, victim_id, servername, host }) => {
-        servername = servername.replace(/[^a-z0-9]/gi, '')
-        const prefix = genPrefix.player({ attacker_id: victim_id, server: `${host}.${servername}` })
-        transaction.json.set('kills', prefix + ".deaths", Number(kills))
-      }))
-      return transaction.exec()
-    })
-
-  //player weapon kills
-  await db.selectFrom('kill').select([count('id').as('kills'), max('distance').as('max_distance'), sum('distance').as('total_distance'), 'attacker_id', 'cause_of_death', 'servername', 'host']).groupBy(['attacker_id', 'cause_of_death', 'servername', 'host']).whereRef("attacker_id", '!=', 'victim_id').execute()
-    .then(async (data) => {
-      let transaction = client.multi()
-      await Promise.all(data.map(async ({ kills, max_distance, total_distance, attacker_id, cause_of_death, servername, host }) => {
-        servername = servername.replace(/[^a-z0-9]/gi, '')
-        await processData(genPrefix.playerWeapons({ attacker_id, cause_of_death, server: `${host}.${servername}` }), { kills, max_distance, total_distance }, transaction)
-        await processData(genPrefix.weaponPlayers({ attacker_id, cause_of_death, server: `${host}.${servername}` }), { kills, max_distance, total_distance }, transaction)
-      }))
-      return transaction.exec()
-    })
-  //player weapon deaths
-  await db.selectFrom('kill').select([count('id').as('kills'), 'victim_id', 'cause_of_death', 'servername', 'host']).groupBy(['victim_id', 'cause_of_death', 'servername', 'host']).execute()
-    .then(async (data) => {
-      let transaction = client.multi()
-      await Promise.all(data.map(async ({ kills, victim_id, cause_of_death, servername, host }) => {
-        servername = servername.replace(/[^a-z0-9]/gi, '')
-        transaction.json.set('kills', genPrefix.weaponPlayers({ attacker_id: victim_id, cause_of_death, server: `${host}.${servername}` }) + ".deaths", Number(kills))
-      }))
-      return transaction.exec()
-    })
-}
-
-async function processData(prefix: string, { total_distance, max_distance, kills }: { total_distance: string | number | bigint, max_distance: number, kills: string | number | bigint }, transaction: ReturnType<typeof client.multi>) {
-  if (!await client.json.type('kills', prefix)) await client.json.set('kills', prefix, {})
-  transaction = transaction.json.set('kills', prefix + ".total_distance", Number(total_distance))
-  transaction = transaction.json.set('kills', prefix + ".max_distance", Number(max_distance))
-  transaction = transaction.json.set('kills', prefix + ".kills", Number(kills))
-  return
-}
-
 export default processAll

src/processMain.ts

@@ -1,14 +0,0 @@
-import * as dotenv from 'dotenv'
-dotenv.config()
-import { dbReady } from './db/db'
-import { cacheReady } from './cache/redis'
-import processAll from './process/process'
-import listenKills from "./process/onKill"
-
-async function startup() {
-  await dbReady()
-  await cacheReady()
-  await listenKills()
-  await processAll()
-}
-export default startup()

src/server/server.ts

@@ -1,6 +1,5 @@
-import { NextFunction, Router } from 'express'
+import { Router } from 'express'
-import expressBasicAuth from 'express-basic-auth'
+import { body, header } from 'express-validator'
-import { body, header, param } from 'express-validator'
 import { CreateKillRecord, CheckServerToken } from '../db/db'
 import { validateErrors } from '../common'
 

tests/client.test.ts

@@ -2,7 +2,6 @@ import { afterAll, beforeAll, describe, expect, test } from '@jest/globals'
 import clientMain from '../src/clientMain'
 import * as dotenv from 'dotenv'
 import db from '../src/db/db'
-import cache from '../src/cache/redis'
 dotenv.config()
 
 let listenServer
@@ -16,9 +15,9 @@ describe('client', () => {
         const request = await fetch("http://127.0.0.1:3000/servers")
         const data = await request.json()
         const first = Object.entries(data)[0]
-        expect(first[1]).toHaveProperty('name')
+        expect(first[1]).toHaveProperty('max_distance')
-        expect(first[1]).toHaveProperty('id')
+        expect(first[1]).toHaveProperty('total_distance')
-        expect(first[1]).toHaveProperty('description')
+        expect(first[1]).toHaveProperty('kills')
     })
 
     test('player list', async () => {
@@ -39,15 +38,6 @@ describe('client', () => {
         expect(first[1]).toHaveProperty('kills')
     })
 
-    test('player list with weapon and server filter', async () => {
-        const request = await fetch("http://127.0.0.1:3000/players?weapons=sniper&server=1")
-        const data = await request.json()
-        const first = Object.entries(data)[0]
-        expect(first[1]).toHaveProperty('max_distance')
-        expect(first[1]).toHaveProperty('total_distance')
-        expect(first[1]).toHaveProperty('kills')
-    })
-
     test('weapon list', async () => {
         const request = await fetch("http://127.0.0.1:3000/weapons")
         const data = await request.json()
@@ -65,21 +55,11 @@ describe('client', () => {
         expect(first[1]).toHaveProperty('total_distance')
         expect(first[1]).toHaveProperty('kills')
     })
-
-    test('weapon list with player and server filter', async () => {
-        const request = await fetch("http://127.0.0.1:3000/weapons?players=1005930844007&server=1")
-        const data = await request.json()
-        const first = Object.entries(data)[0]
-        expect(first[1]).toHaveProperty('max_distance')
-        expect(first[1]).toHaveProperty('total_distance')
-        expect(first[1]).toHaveProperty('kills')
-    })
 })
 
 afterAll((done) => {
     listenServer.close(async () => {
         await db.destroy()
-        await cache.quit()
         done()
     })
 })

tests/realtime.test.ts

@@ -4,7 +4,6 @@ import serverMain from '../src/serverMain'
 import listenKills from "../src/process/onKill"
 import * as dotenv from 'dotenv'
 import db from '../src/db/db'
-import cache from '../src/cache/redis'
 dotenv.config()
 
 let listenClient
@@ -62,8 +61,8 @@ beforeAll(async () => {
     listenClient = await clientMain;
     listenServer = await serverMain;
     pgClient = await listenKills()
-    const yea = waitFor(10000)
+    const yea = waitFor(1000)
-    const response = await fetch(`http://127.0.0.1:3001/${process.env.SERVERAUTH_ID}/kill`, {
+    const response = await fetch(`http://127.0.0.1:3001/kill`, {
         method: "POST", // *GET, POST, PUT, DELETE, etc.
         credentials: "same-origin", // include, *same-origin, omit
         headers: {
@@ -103,8 +102,8 @@ describe('realtime', () => {
 
     test('update player', async () => {
         jest.setTimeout(15000)
-        const yea = waitFor(10000)
+        const yea = waitFor(1000)
-        const response = await fetch(`http://127.0.0.1:3001/${process.env.SERVERAUTH_ID}/kill`, {
+        const response = await fetch(`http://127.0.0.1:3001/kill`, {
             method: "POST", // *GET, POST, PUT, DELETE, etc.
             credentials: "same-origin", // include, *same-origin, omit
             headers: {
@@ -138,7 +137,6 @@ afterAll((done) => {
         listenServer.close(async () => {
             await pgClient.end()
             await db.destroy()
-            await cache.quit()
             done()
         })
     })