Big refactor to allow for more data processing

This commit is contained in:
2023-03-14 21:47:48 +01:00
parent 2808663aaa
commit 0c8513da1f
8 changed files with 357 additions and 355 deletions
+26
View File
@@ -0,0 +1,26 @@
import cache from './redis'
/**
* Shows the data for one weapon
* @param weapon
* @param server
* @param player
* @returns
*/
export async function getWeaponReport(
weapon: string,
server?: number,
player?: number
) {
const serverPrefix = server ? `servers:${server}:` : ''
const playerPrefix = player ? `players:${player}:` : ''
const cacheLocation = serverPrefix + playerPrefix + `weapons:` + weapon
const data = await cache.HGETALL(cacheLocation)
delete data.last_entry
return data
}
export function getWeaponList(server?: string, player?: string) {
const serverPrefix = server ? `servers:${server}:` : ''
const playerPrefix = player ? `players:${player}:` : ''
return cache.GET(serverPrefix + playerPrefix + `weapons:processedList`)
}
+70 -6
View File
@@ -1,27 +1,91 @@
import { Router } from 'express'
import serverWeapons from './routes/serverWeapons'
import playerWeapons from './routes/playerWeapons'
import { param } from 'express-validator'
import { validateErrors } from '../common'
import serverPlayers from './routes/serverPlayers'
import serverList from './routes/serverList'
import player from './routes/player'
import { getWeaponList, getWeaponReport } from '../cache/cacheUtils'
const router = Router()
//timeout middleware ?
router.get('/*', (req, res, next) => {
next()
})
//router.get('/weapons/', (req, res, next) => {})
router.get('/weapons/', async (req, res) => {
const data = JSON.parse((await getWeaponList()) || '{}')
res.status(200).send(data)
})
router.get(
'/weapons/:weaponId',
param(['weaponId']).exists().isString(),
validateErrors,
async (req, res) => {
const weapon = req.params.weaponId
const data = await getWeaponReport(weapon)
res.status(200).send(data)
}
)
//router.get('/maps/', (req, res, next) => {})
//router.get('/players', (req, res, next) => {})
//router.get('/players/:playerId', (req, res, next) => {})
router.get('/players/:playerId/weapons')
router.get('/servers/', serverList)
//router.get('/servers/:serverId/', (req, res, next) => {})
router.get('/servers/:serverId/players', serverPlayers)
router.get('/servers/:serverId/players/:playerId', player)
router.get('/servers/:serverId/players/:playerId/weapons', playerWeapons)
router.use('/servers/:serverId/weapons', serverWeapons)
//router.get('/servers/:serverId/players/:playerId', player)
router.get(
'/servers/:serverId/players/:playerId/weapons',
param(['serverId', 'playerId']).exists().toInt().isInt(),
validateErrors,
async (req, res) => {
const data = JSON.parse(
(await getWeaponList(req.params.serverId, req.params.playerId)) || '{}'
)
res.status(200).send(data)
}
)
router.get(
'/servers/:serverId/players/:playerId/weapons:weaponId',
param(['serverId', 'playerId, weaponId']).exists().toInt().isInt(),
validateErrors,
async (req, res) => {
const data = getWeaponReport(
req.params.weaponId,
Number(req.params.serverId),
Number(req.params.playerId)
)
res.status(200).send(data)
}
)
router.use(
'/servers/:serverId/weapons',
param(['serverId']).exists().toInt().isInt(),
validateErrors,
async (req, res) => {
const data = JSON.parse((await getWeaponList(req.params.serverId)) || '{}')
res.status(200).send(data)
}
)
router.use(
'/servers/:serverId/weapons',
param(['serverId']).exists().toInt().isInt(),
validateErrors,
async (req, res) => {
const data = getWeaponReport(
req.params.weaponId,
Number(req.params.serverId)
)
res.status(200).send(data)
}
)
export default router
+75 -13
View File
@@ -1,42 +1,104 @@
import db from '../db/db'
import cache from '../cache/redis'
import { processServerPlayers } from './routes/serverPlayers'
import { processServerWeapons } from './routes/serverWeapons'
import { processPlayerWeapons } from './routes/playerWeapons'
/*import { processPlayerWeapons } from './routes/serverPlayerWeapons'*/
import { processPlayerReport } from './routes/player'
import {
processWeaponReport,
populateWeaponSet,
processWeaponList
} from './process/processWeapon'
/**
* Starts the process of populating the REDIS database globally
* @returns
*/
async function processAll() {
console.log('Starting data calculation...')
const timeStart = new Date()
// Iterate through all servers
const servers = await db.selectFrom('server').selectAll().execute()
const promises: Promise<any>[] = []
servers.forEach((e) => {
promises.push(processServer(e.id))
servers.forEach((server) => {
promises.push(processServer(server.id))
})
//Iterate through all weapons
await populateWeaponSet()
const weapons = await cache.SMEMBERS('weapons')
weapons.forEach((weapon) => {
promises.push(processWeaponReport(weapon))
})
await Promise.all(promises)
await processWeaponList()
//TODO : iterate through all players
console.log(
'Data calculation finished. Took + ' +
Math.abs(new Date().getTime() - timeStart.getTime()) / 1000 +
' seconds'
)
if (process.env.ENVIRONMENT == 'production') {
return
}
setTimeout(processAll, 60000)
}
/**
* Starts the process of populating the REDIS database for server-related things
* @param server
*/
async function processServer(server: number) {
await processServerPlayers(server)
const promises: Promise<any>[] = []
promises.push(processServerWeapons(server))
//TODO TO REMOVE OR RENAME
await processServerPlayers(server)
//Iterate through all weapons present on this server
await populateWeaponSet(server)
const weapons = await cache.SMEMBERS(`servers:${server}:weapons`)
weapons.forEach((weapon) => {
promises.push(processWeaponReport(weapon, server))
})
//Iterate through all players present on this server (and do more stuff on those players)
const players = await cache.SMEMBERS(`servers:${server}:players`)
players.forEach((player) => {
promises.push(processPlayer(server, Number(player)))
promises.push(processPlayer({ server, player: Number(player) }))
})
await Promise.all(promises)
await processWeaponList(server)
}
/**
* Starts the process of populating the REDIS database for player-related things
* @param param0
*/
async function processPlayer({
server,
player
}: {
server?: number
player: number
}) {
const serverPrefix = server ? `servers:${server}:` : ''
const playerPrefix = player ? `players:${player}:` : ''
const cacheLocation = serverPrefix + playerPrefix + `weapons`
const promises: Promise<any>[] = []
//Iterate through all weapons for this player
await populateWeaponSet(server, player)
const weapons = await cache.SMEMBERS(cacheLocation)
weapons.forEach((weapon) => {
promises.push(processWeaponReport(weapon, server, player))
})
await Promise.all(promises)
await processWeaponList(server, player)
//TODO : more data analytics
/*
promises.push(processPlayerWeapons(server, player))
promises.push(processPlayerReport(server, player))*/
await Promise.all(promises)
}
async function processPlayer(server: number, player: number) {
const promises: Promise<any>[] = []
promises.push(processPlayerWeapons(server, player))
promises.push(processPlayerReport(server, player))
}
export default processAll
+181
View File
@@ -0,0 +1,181 @@
/**
* This file contains all functions related to population of the REDIS database for weapons.
*/
import db from '../../db/db'
import cache from '../../cache/redis'
import { getWeaponReport } from '../../cache/cacheUtils'
const { count, max, avg } = db.fn
/**
* Populates the set of weapons (just a list with weapon names)
*
* Inputs `POSTGRESQL`
*
* Outputs `SET` `[servers:{serverId}:][players:{playerId}:]weapons`
* @param server optional server filter
* @param player optional player filter
* @returns
*/
export async function populateWeaponSet(server?: number, player?: number) {
const serverPrefix = server ? `servers:${server}:` : ''
const playerPrefix = player ? `players:${player}:` : ''
const cacheLocation = serverPrefix + playerPrefix + `weapons`
const last_entry = Number(await cache.GET(cacheLocation + ':last_entry')) || 0
let query = db
.selectFrom('kill')
.select([
'cause_of_death',
db
.selectFrom('kill')
.select(max('kill.id').as('last_entry'))
.as('last_entry')
])
.groupBy('cause_of_death')
.where('kill.id', '>', last_entry)
if (server) {
query = query.where('server', '=', server)
}
if (player) {
query = query
.where('attacker_id', '=', player.toString())
.orWhere('victim_id', '=', player.toString())
}
const newData = await query.execute()
if (newData.length == 0) {
return
}
const promises: Promise<any>[] = []
newData.forEach(({ cause_of_death }) => {
if (cause_of_death == 'last_entry' || cause_of_death == 'processedList') {
console.error('cause_of_death cannot be ' + cause_of_death)
return
}
promises.push(cache.SADD(cacheLocation, cause_of_death))
})
promises.push(cache.SET(cacheLocation + ':last_entry', newData[0].last_entry))
await Promise.all(promises)
}
/**
* Processes weapon kills with optional server and/or player filter.
* Does not processes deaths from weapons.
*
* Inputs `POSTGRESQL`
*
* Outputs `HASH` `[servers:{serverId}:][players:{playerId}:]weapons:{weaponId}`
* @param weapon weapon to generate the report for
* @param server optional server filter
* @param player optional player filter
*/
export async function processWeaponReport(
weapon: string,
server?: number,
player?: number
) {
if (weapon == 'last_entry' || weapon == 'processedList') {
console.error('Weapon cannot be ' + weapon)
return
}
const serverPrefix = server ? `servers:${server}:` : ''
const playerPrefix = player ? `players:${player}:` : ''
const cacheLocation = serverPrefix + playerPrefix + `weapons:` + weapon
const last_entry = Number(await cache.HGET(cacheLocation, 'last_entry')) || 0
let query = db
.selectFrom('kill')
.select([
count<number>('id').as('kills'),
avg<number>('kill.distance').as('avg_kill_distance'),
max('kill.distance').as('max_kill_distance'),
db
.selectFrom('kill')
.select(max('kill.id').as('last_entry'))
.as('last_entry')
])
.where('cause_of_death', '=', weapon)
.where('kill.id', '>', last_entry)
.groupBy('cause_of_death')
if (server) {
query = query.where('server', '=', server)
}
if (player) {
query = query.where('attacker_id', '=', player.toString())
}
const newData = await query.execute()
if (newData.length == 0) {
return
}
const promises: Promise<any>[] = []
newData.forEach(({ kills, avg_kill_distance, max_kill_distance }) => {
promises.push(
processAvg(cacheLocation, 'avg_kill_distance', {
newkills: kills,
newavg: avg_kill_distance
}).then((e) => cache.HINCRBY(cacheLocation, 'kills', kills)),
processMax(cacheLocation, 'max_kill_distance', max_kill_distance)
)
})
promises.push(cache.HSET(cacheLocation, 'last_entry', newData[0].last_entry))
await Promise.all(promises)
}
async function processAvg(
cacheLocation: string,
key: 'avg_kill_distance',
{ newkills, newavg }: { newkills: number; newavg: number }
) {
const oldKills = Number(await cache.HGET(cacheLocation, 'kills')) || 0
const oldaverage = Number(await cache.HGET(cacheLocation, key)) | 0
const totalkills = oldKills + newkills
const newAverage =
(oldKills / totalkills) * oldaverage + (newkills / totalkills) * newavg
await cache.HSET(cacheLocation, key, newAverage)
}
async function processMax(
cacheLocation: string,
key: 'max_kill_distance',
newmax: number
) {
const oldMax = Number(await cache.HGET(cacheLocation, key)) | 0
if (oldMax < newmax) await cache.HSET(cacheLocation, key, newmax)
}
/**
* Combines weapons reports in a readable json string.
*
* ProcessWeaponReports must be called manually before this function !
*
* Inputs `SET` `[servers:{serverId}:][players:{playerId}:]weapons`
*
* Outputs `STRING` `[servers:{serverId}:][players:{playerId}:]weapons:processedList`
* @param server optional server filter
* @param player optional player filter
* @returns
*/
export async function processWeaponList(server?: number, player?: number) {
const serverPrefix = server ? `servers:${server}:` : ''
const playerPrefix = player ? `players:${player}:` : ''
const weapons = await cache.SMEMBERS('weapons')
const promises: Promise<any>[] = []
const data: {
[weaponId: string]: {
[weaponData: string]: string
}
} = {}
weapons.forEach((wpn) => {
promises.push(
(async () => {
const weaponData = await getWeaponReport(wpn, server, player)
if (Object.keys(weaponData).length > 0) {
data[wpn] = weaponData
}
})()
)
})
await Promise.all(promises)
return await cache.SET(
serverPrefix + playerPrefix + `weapons:processedList`,
JSON.stringify(data)
)
}
+5 -3
View File
@@ -4,8 +4,9 @@ import { validateErrors } from '../../common'
import cache from '../../cache/redis'
import { sql } from 'kysely'
import db from '../../db/db'
import { getPlayerWeapons } from './playerWeapons'
//import { getPlayerWeapons } from './serverPlayerWeapons'
//TODO : NOT WORKING AS OF NOW
const { count, max, min, avg } = db.fn
const middlewares: RequestHandler[] = [
@@ -16,8 +17,9 @@ const middlewares: RequestHandler[] = [
const player = Number(req.params.playerId)
//await processPlayerReport(server, player)
const data = await getPlayerReport(server, player)
const weapons = await getPlayerWeapons(server, player)
res.status(200).send({ ...data, weapons })
//const weapons = await getPlayerWeapons(server, player)
res.status(200).send(data)
//res.status(200).send({ ...data, weapons })
}
]
-186
View File
@@ -1,186 +0,0 @@
import { RequestHandler } from 'express'
import { param } from 'express-validator'
import { validateErrors } from '../../common'
import cache from '../../cache/redis'
import { sql } from 'kysely'
import db from '../../db/db'
const { count, max, avg } = db.fn
const middlewares: RequestHandler[] = [
param(['serverId', 'playerId']).exists().toInt().isInt(),
validateErrors,
async (req, res) => {
const server = Number(req.params.serverId)
const player = Number(req.params.playerId)
//await processPlayerWeapons(server, player)
const data = await getPlayerWeapons(server, player)
res.status(200).send(data)
}
]
export async function getPlayerWeapons(server: number, player: number) {
const killsData = await cache.HGETALL(
`servers:${server}:players:${player}:weapons:kills`
)
const deathData = await cache.HGETALL(
`servers:${server}:players:${player}:weapons:deaths`
)
const avg_distance = await cache.HGETALL(
`servers:${server}:players:${player}:weapons:avg_kill_distance`
)
const max_distance = await cache.HGETALL(
`servers:${server}:players:${player}:weapons:max_kill_distance`
)
const data: {
[x: string]: {
kills: string
deaths: string
avg_kill_distance: string
max_kill_distance: string
}
} = {}
Object.keys(killsData).forEach((key) => {
data[key] = {
kills: killsData[key],
deaths: deathData[key],
avg_kill_distance: avg_distance[key],
max_kill_distance: max_distance[key]
}
})
return data
}
export async function processPlayerWeapons(server: number, player: number) {
let last_entry =
Number(
await cache.HGET(
`servers:${server}:players:${player}:weapons`,
'last_entry'
)
) || 0
const newData = await db
.with('killdata', () =>
db
.selectFrom('kill')
.select([
count('kill.id').as('num_kills'),
'kill.cause_of_death',
max('distance').as('max_kill_distance'),
avg('distance').as('avg_kill_distance')
])
.where('attacker_id', '=', player.toString())
.where('kill.id', '>', last_entry)
.where('kill.server', '=', server)
.whereRef('attacker_id', '!=', 'victim_id')
.groupBy('cause_of_death')
)
.with('deathdata', () =>
db
.selectFrom('kill')
.select([count('kill.id').as('num_deaths'), 'kill.cause_of_death'])
.where('victim_id', '=', player.toString())
.where('kill.id', '>', last_entry)
.where('kill.server', '=', server)
.groupBy('cause_of_death')
)
.selectFrom('killdata')
.select([
sql<number>`coalesce(killdata.num_kills, 0)`.as('kills'),
sql<number>`coalesce(deathdata.num_deaths, 0)`.as('deaths'),
sql<string>`COALESCE(killdata.cause_of_death, deathdata.cause_of_death)`.as(
'weapon'
),
'killdata.avg_kill_distance',
'killdata.max_kill_distance',
db
.selectFrom('kill')
.select(max('kill.id').as('last_entry'))
.as('last_entry')
])
.fullJoin(
'deathdata',
'deathdata.cause_of_death',
'killdata.cause_of_death'
)
.orderBy('kills', 'desc')
.execute()
//if no new kills
if (newData.length == 0) {
return
}
last_entry = newData.reduce(
(acc, current) => (current.last_entry > acc ? current.last_entry : acc),
0
)
const promises: Promise<any>[] = []
newData.forEach(
async ({ weapon, kills, deaths, avg_kill_distance, max_kill_distance }) => {
promises.push(
(async () => {
const oldKills =
Number(
await cache.HGET(
`servers:${server}:players:${player}:weapons:kills`,
weapon
)
) || 0
const oldaverage =
Number(
await cache.HGET(
`servers:${server}:players:${player}:weapons:avg_kill_distance`,
weapon
)
) | 0
const totalkills = oldKills + kills
const newAverage =
(oldKills / totalkills) * oldaverage +
(kills / totalkills) * Number(avg_kill_distance)
const oldMaxDistance =
Number(
await cache.HGET(
`servers:${server}:players:${player}:weapons:max_kill_distance`,
weapon
)
) | 0
promises.push(
cache.HINCRBY(
`servers:${server}:players:${player}:weapons:deaths`,
weapon,
deaths
),
cache.HINCRBY(
`servers:${server}:players:${player}:weapons:kills`,
weapon,
kills
),
cache.HSET(
`servers:${server}:players:${player}:weapons:avg_kill_distance`,
weapon,
newAverage | 0
),
cache.HSET(
`servers:${server}:players:${player}:weapons:max_kill_distance`,
weapon,
oldMaxDistance < max_kill_distance
? max_kill_distance
: oldMaxDistance
)
)
})()
)
}
)
promises.push(
cache.HSET(
`servers:${server}:players:${player}:weapons`,
'last_entry',
newData[0].last_entry
)
)
await Promise.all(promises)
}
export default middlewares
-1
View File
@@ -2,7 +2,6 @@ import { RequestHandler } from 'express'
import { param } from 'express-validator'
import { validateErrors } from '../../common'
import cache from '../../cache/redis'
import { sql } from 'kysely'
import db from '../../db/db'
const { count, max } = db.fn
-146
View File
@@ -1,146 +0,0 @@
import { RequestHandler } from 'express'
import { param } from 'express-validator'
import { validateErrors } from '../../common'
import cache from '../../cache/redis'
import { sql } from 'kysely'
import db from '../../db/db'
const { count, max, avg } = db.fn
const middlewares: RequestHandler[] = [
param(['serverId']).exists().toInt().isInt(),
validateErrors,
async (req, res) => {
const server = Number(req.params.serverId)
//await processServerWeapons(server)
const data = await getServerWeapons(server)
res.status(200).send(data)
}
]
export async function getServerWeapons(server: number) {
const killsData = await cache.HGETALL(`servers:${server}:weapons:kills`)
const avg_distance = await cache.HGETALL(
`servers:${server}:weapons:avg_kill_distance`
)
const max_distance = await cache.HGETALL(
`servers:${server}:weapons:max_kill_distance`
)
const data: {
[x: string]: {
kills: string
avg_kill_distance: string
max_kill_distance: string
}
} = {}
Object.keys(killsData).forEach((key) => {
data[key] = {
kills: killsData[key],
avg_kill_distance: avg_distance[key],
max_kill_distance: max_distance[key]
}
})
return data
}
export async function processServerWeapons(server: number) {
let last_entry =
Number(await cache.HGET(`servers:${server}:weapons`, 'last_entry')) || 0
const newData = await db
.with('killdata', () =>
db
.selectFrom('kill')
.select([
count('kill.id').as('num_kills'),
'kill.cause_of_death',
max('distance').as('max_kill_distance'),
avg('distance').as('avg_kill_distance')
])
.where('kill.id', '>', last_entry)
.where('kill.server', '=', server)
.whereRef('attacker_id', '!=', 'victim_id')
.groupBy('cause_of_death')
)
.selectFrom('killdata')
.select([
sql<number>`coalesce(killdata.num_kills, 0)`.as('kills'),
'killdata.cause_of_death',
'killdata.avg_kill_distance',
'killdata.max_kill_distance',
db
.selectFrom('kill')
.select(max('kill.id').as('last_entry'))
.as('last_entry')
])
.orderBy('kills', 'desc')
.execute()
//if no new kills
if (newData.length == 0) {
return
}
last_entry = newData.reduce(
(acc, current) => (current.last_entry > acc ? current.last_entry : acc),
0
)
const promises: Promise<any>[] = []
newData.forEach(
({ cause_of_death, kills, avg_kill_distance, max_kill_distance }) => {
promises.push(
(async () => {
const oldKills =
Number(
await cache.HGET(
`servers:${server}:weapons:kills`,
cause_of_death
)
) || 0
const oldaverage =
Number(
await cache.HGET(
`servers:${server}:weapons:avg_kill_distance`,
cause_of_death
)
) | 0
const totalkills = oldKills + kills
const newAverage =
(oldKills / totalkills) * oldaverage +
(kills / totalkills) * Number(avg_kill_distance)
const oldMaxDistance =
Number(
await cache.HGET(
`servers:${server}:weapons:max_kill_distance`,
cause_of_death
)
) | 0
promises.push(
cache.HINCRBY(
`servers:${server}:weapons:kills`,
cause_of_death,
kills
),
cache.HSET(
`servers:${server}:weapons:avg_kill_distance`,
cause_of_death,
newAverage | 0
),
cache.HSET(
`servers:${server}:weapons:max_kill_distance`,
cause_of_death,
oldMaxDistance < max_kill_distance
? max_kill_distance
: oldMaxDistance
)
)
})()
)
}
)
promises.push(
cache.HSET(`servers:${server}:weapons`, 'last_entry', newData[0].last_entry)
)
await Promise.all(promises)
}
export default middlewares