big refactor for lots of filtering

This commit is contained in:
2023-03-15 17:29:48 +01:00
parent c57e7fa556
commit afdd4f3fc6
7 changed files with 488 additions and 130 deletions
+61 -3
View File
@@ -9,7 +9,7 @@ import cache from './redis'
export async function getWeaponReport(
weapon: string,
server?: number,
player?: number
player?: string
) {
const serverPrefix = server ? `servers:${server}:` : ''
const playerPrefix = player ? `players:${player}:` : ''
@@ -19,8 +19,66 @@ export async function getWeaponReport(
return data
}
export function getWeaponList(server?: string, player?: string) {
export function getWeaponSet(server?: number, player?: string) {
const serverPrefix = server ? `servers:${server}:` : ''
const playerPrefix = player ? `players:${player}:` : ''
return cache.GET(serverPrefix + playerPrefix + `weapons:processedList`)
const cacheLocation = serverPrefix + playerPrefix + `weapons`
return cache.SMEMBERS(cacheLocation)
}
/**
* Show the processed list of weapons
* @param server
* @param player
* @returns string
*/
export async function getWeaponList(server?: string, player?: string) {
const serverPrefix = server ? `servers:${server}:` : ''
const playerPrefix = player ? `players:${player}:` : ''
return JSON.parse(
(await cache.GET(serverPrefix + playerPrefix + `weapons:processedList`)) ||
'{}'
)
}
/**
* Show processed data for a player
* @param player
* @param server
* @param weapon
* @returns
*/
export async function getPlayerReport(
player: string,
server?: number,
weapon?: string
) {
const serverPrefix = server ? `servers:${server}:` : ''
const weaponPrefix = weapon ? `weapons:${weapon}:` : ''
const cacheLocation = serverPrefix + weaponPrefix + `players:` + player
const data = await cache.HGETALL(cacheLocation)
delete data.last_entry
return data
}
/**
* Shows processed list of players
* @param server
* @param weapon
* @returns string
*/
export async function getPlayerList(server?: number, weapon?: string) {
const serverPrefix = server ? `servers:${server}:` : ''
const weaponPrefix = weapon ? `weapons:${weapon}:` : ''
return JSON.parse(
(await cache.GET(serverPrefix + weaponPrefix + `players:processedList`)) ||
'{}'
)
}
export function getPlayerSet(server?: number, weapon?: string) {
const serverPrefix = server ? `servers:${server}:` : ''
const weaponPrefix = weapon ? `weapons:${weapon}:` : ''
const cacheLocation = serverPrefix + weaponPrefix + `players`
return cache.SMEMBERS(cacheLocation)
}
+69 -67
View File
@@ -1,91 +1,93 @@
import { Router } from 'express'
import { param } from 'express-validator'
import { param, query } from 'express-validator'
import { validateErrors } from '../common'
import serverPlayers from './routes/serverPlayers'
import serverList from './routes/serverList'
import player from './routes/player'
import { getWeaponList, getWeaponReport } from '../cache/cacheUtils'
import {
getWeaponList,
getWeaponReport,
getPlayerReport,
getPlayerList
} from '../cache/cacheUtils'
const router = Router()
//timeout middleware ?
router.get('/*', (req, res, next) => {
next()
})
router.get('/weapons/', async (req, res) => {
const data = JSON.parse((await getWeaponList()) || '{}')
res.status(200).send(data)
})
router.get(
'/weapons/:weaponId',
param(['weaponId']).exists().isString(),
query(['player', 'server']).optional().toInt().isInt(),
validateErrors,
async (req, res) => {
const weapon = req.params.weaponId
const data = await getWeaponReport(weapon)
const data = await getWeaponReport(
req.params.weaponId,
Number(req.query.server) || undefined,
req.query.player?.toString()
)
res.status(200).send(data)
}
)
router.get(
'/weapons/',
query(['player', 'server']).optional().toInt().isInt(),
validateErrors,
async (req, res) => {
const data = await getWeaponList(
req.query.server?.toString(),
req.query.player?.toString()
)
res.status(200).send(data)
}
)
router.get(
'/players/:playerId',
param(['playerId']).exists().toInt().isInt(),
query(['server']).optional().toInt().isInt(),
query('weapon').optional().isString(),
validateErrors,
async (req, res) => {
const data = await getPlayerReport(
req.params.playerId,
Number(req.query.server) || undefined,
req.query.weapon?.toString()
)
if (!req.query.weapon) {
const weapons = await getWeaponList(
req.query.server?.toString(),
req.params.playerId
)
data.weapons = weapons
} else {
data.weapons = (await getWeaponReport(
req.query.weapon.toString(),
Number(req.query.server) || undefined,
req.params.playerId
)) as any
}
res.status(200).send(data)
}
)
router.get(
'/players/',
query(['server']).optional().toInt().isInt(),
query('weapon').optional().isString(),
validateErrors,
async (req, res) => {
const data = await getPlayerList(
Number(req.query.server) || undefined,
req.query.weapon?.toString()
)
res.status(200).send(data)
}
)
//router.get('/maps/', (req, res, next) => {})
//router.get('/players', (req, res, next) => {})
//router.get('/players/:playerId', (req, res, next) => {})
router.get('/players/:playerId/weapons')
router.get('/servers/', serverList)
//router.get('/servers/:serverId/', (req, res, next) => {})
router.get('/servers/:serverId/players', serverPlayers)
//router.get('/servers/:serverId/players/:playerId', player)
router.get(
'/servers/:serverId/players/:playerId/weapons',
param(['serverId', 'playerId']).exists().toInt().isInt(),
validateErrors,
async (req, res) => {
const data = JSON.parse(
(await getWeaponList(req.params.serverId, req.params.playerId)) || '{}'
)
res.status(200).send(data)
}
)
router.get(
'/servers/:serverId/players/:playerId/weapons:weaponId',
param(['serverId', 'playerId, weaponId']).exists().toInt().isInt(),
validateErrors,
async (req, res) => {
const data = getWeaponReport(
req.params.weaponId,
Number(req.params.serverId),
Number(req.params.playerId)
)
res.status(200).send(data)
}
)
router.use(
'/servers/:serverId/weapons',
param(['serverId']).exists().toInt().isInt(),
validateErrors,
async (req, res) => {
const data = JSON.parse((await getWeaponList(req.params.serverId)) || '{}')
res.status(200).send(data)
}
)
router.use(
'/servers/:serverId/weapons',
param(['serverId']).exists().toInt().isInt(),
validateErrors,
async (req, res) => {
const data = getWeaponReport(
req.params.weaponId,
Number(req.params.serverId)
)
res.status(200).send(data)
}
)
export default router
+142 -35
View File
@@ -2,12 +2,17 @@ import db from '../db/db'
import cache from '../cache/redis'
import { processServerPlayers } from './routes/serverPlayers'
/*import { processPlayerWeapons } from './routes/serverPlayerWeapons'*/
import { processPlayerReport } from './routes/player'
import {
processWeaponReport,
populateWeaponSet,
processWeaponList
} from './process/processWeapon'
import {
populatePlayerSet,
processPlayerList,
processPlayerReport
} from './process/processPlayer'
import { getWeaponSet, getPlayerSet } from '../cache/cacheUtils'
/**
* Starts the process of populating the REDIS database globally
@@ -16,24 +21,29 @@ import {
async function processAll() {
console.log('Starting data calculation...')
const timeStart = new Date()
// Iterate through all servers
const servers = await db.selectFrom('server').selectAll().execute()
await populateAllSets()
const promises: Promise<any>[] = []
servers.forEach((server) => {
promises.push(processServer(server.id))
})
//Iterate through all weapons
await populateWeaponSet()
const weapons = await cache.SMEMBERS('weapons')
const weapons = await getWeaponSet()
weapons.forEach((weapon) => {
promises.push(processWeaponReport(weapon))
promises.push(processWeapon(weapon))
})
//Iterate through all players
const players = await getPlayerSet()
players.forEach((player) => {
promises.push(processPlayer(player))
})
await Promise.all(promises)
await processWeaponList()
//Generate lists now that we have all data
await processAllLists()
//TODO : iterate through all players
console.log(
'Data calculation finished. Took + ' +
Math.abs(new Date().getTime() - timeStart.getTime()) / 1000 +
@@ -46,11 +56,11 @@ async function processAll() {
}
/**
* Starts the process of populating the REDIS database for server-related things
* Populte redis of server lists and reports
* @param server
*/
async function processServer(server: number) {
const promises: Promise<any>[] = []
/*const promises: Promise<any>[] = []
//TODO TO REMOVE OR RENAME
await processServerPlayers(server)
@@ -64,41 +74,138 @@ async function processServer(server: number) {
//Iterate through all players present on this server (and do more stuff on those players)
const players = await cache.SMEMBERS(`servers:${server}:players`)
players.forEach((player) => {
promises.push(processPlayer({ server, player: Number(player) }))
promises.push(processPlayer({ server, player }))
})
await Promise.all(promises)
await processWeaponList(server)
await processWeaponList(server)*/
}
/**
* Starts the process of populating the REDIS database for player-related things
* Populate redis of player lists and reports
* @param param0
*/
async function processPlayer({
server,
player
}: {
server?: number
player: number
}) {
const serverPrefix = server ? `servers:${server}:` : ''
const playerPrefix = player ? `players:${player}:` : ''
const cacheLocation = serverPrefix + playerPrefix + `weapons`
async function processPlayer(player: string) {
const promises: Promise<any>[] = []
//Iterate through all weapons for this player
await populateWeaponSet(server, player)
const weapons = await cache.SMEMBERS(cacheLocation)
weapons.forEach((weapon) => {
promises.push(processWeaponReport(weapon, server, player))
//iterate through all servers
const servers = await db.selectFrom('server').selectAll().execute()
servers.forEach((server) => {
promises.push(
processPlayerReport(player, server.id),
//Iterate through all players on servers
getWeaponSet(server.id, player).then((weapons) => {
weapons.forEach((weapon) => {
promises.push(processPlayerReport(player, server.id, weapon))
})
})
)
})
await Promise.all(promises)
await processWeaponList(server, player)
//TODO : more data analytics
/*
promises.push(processPlayerWeapons(server, player))
promises.push(processPlayerReport(server, player))*/
//iterate through all weapons outside of servers
const weapons = await getWeaponSet(undefined, player)
weapons.forEach((weapon) => {
promises.push(processPlayerReport(player, undefined, weapon))
})
//global weapon data
promises.push(processPlayerReport(player))
await Promise.all(promises)
}
async function processWeapon(weapon: string) {
const promises: Promise<any>[] = []
//iterate through all servers
const servers = await db.selectFrom('server').selectAll().execute()
servers.forEach((server) => {
promises.push(
processWeaponReport(weapon, server.id),
//Iterate through all players on servers
getPlayerSet(server.id, weapon).then((players) => {
players.forEach((player) => {
promises.push(processWeaponReport(weapon, server.id, player))
})
})
)
})
//iterate through all players outside of servers
const players = await getPlayerSet(undefined, weapon)
players.forEach((player) => {
promises.push(processWeaponReport(weapon, undefined, player))
})
//global weapon data
promises.push(processWeaponReport(weapon))
await Promise.all(promises)
}
async function populateAllSets() {
const servers = await db.selectFrom('server').selectAll().execute()
await Promise.all([populatePlayerSet(), populateWeaponSet()])
await (async () => {
const promises: Promise<any>[] = []
await getWeaponSet().then((weaponSet) => {
weaponSet.forEach((weapon: string) => {
promises.push(populatePlayerSet(undefined, weapon))
})
})
await getPlayerSet().then((playerSet) => {
playerSet.forEach((player) => {
promises.push(populateWeaponSet(undefined, player))
})
})
return Promise.all(promises)
})()
await (async () => {
const promises: Promise<any>[] = servers.map((server) =>
(async () => {
await Promise.all([
populatePlayerSet(server.id),
populateWeaponSet(server.id)
])
const promises: Promise<any>[] = []
await getWeaponSet(server.id).then((weaponSet) => {
weaponSet.forEach((weapon: string) => {
promises.push(populatePlayerSet(server.id, weapon))
})
})
await getPlayerSet(server.id).then((playerSet) => {
playerSet.forEach((player) => {
promises.push(populateWeaponSet(server.id, player))
})
})
return Promise.all(promises)
})()
)
await Promise.all(promises)
})()
}
async function processAllLists() {
const promises: Promise<any>[] = []
promises.push(processPlayerList(), processWeaponList())
promises.push(
...(await getPlayerSet()).map((player) =>
processWeaponList(undefined, player)
),
...(await (
await getWeaponSet()
).map((weapon) => processPlayerList(undefined, weapon)))
)
const servers = await db.selectFrom('server').selectAll().execute()
promises.push(
...servers.map(async ({ id }) => {
const promises: Promise<any>[] = []
promises.push(
processWeaponList(id),
processPlayerList(id),
...(await getPlayerSet()).map((player) =>
processWeaponList(id, player)
),
...(await (
await getWeaponSet()
).map((weapon) => processPlayerList(id, weapon)))
)
await Promise.all(promises)
})
)
await Promise.all(promises)
}
export default processAll
+189
View File
@@ -0,0 +1,189 @@
/**
* This file contains all functions related to population of the REDIS database for players.
*/
import db from '../../db/db'
import cache from '../../cache/redis'
import { getPlayerReport, getPlayerSet } from '../../cache/cacheUtils'
const { count, max, avg, sum } = db.fn
/**
* Populates the set of players (just a list with player IDS)
*
* Inputs `POSTGRESQL`
*
* Outputs `SET` `[servers:{serverId}:][weapons:{weaponId}:]players`
* @param server optional server filter
* @param weapon optional weapon filter
* @returns
*/
export async function populatePlayerSet(server?: number, weapon?: string) {
const serverPrefix = server ? `servers:${server}:` : ''
const weaponPrefix = weapon ? `weapons:${weapon}:` : ''
const cacheLocation = serverPrefix + weaponPrefix + `players`
const last_entry = Number(await cache.GET(cacheLocation + ':last_entry')) || 0
let query = db
.selectFrom('kill')
.select([
'attacker_id',
db
.selectFrom('kill')
.select(max('kill.id').as('last_entry'))
.as('last_entry')
])
.groupBy('attacker_id')
.where('kill.id', '>', last_entry)
if (server) {
query = query.where('server', '=', server)
}
if (weapon) {
query = query.where('cause_of_death', '=', weapon)
}
const newData = await query.execute()
if (newData.length == 0) {
return
}
const promises: Promise<any>[] = []
newData.forEach(({ attacker_id }) => {
if (attacker_id == 'last_entry' || attacker_id == 'processedList') {
console.error('attacker_id cannot be ' + attacker_id)
return
}
promises.push(cache.SADD(cacheLocation, attacker_id))
})
promises.push(cache.SET(cacheLocation + ':last_entry', newData[0].last_entry))
await Promise.all(promises)
}
/**
* Processes player kills with optional server and/or weapon filter.
* Does not processes deaths from weapons.
*
* Inputs `POSTGRESQL`
*
* Outputs `HASH` `[servers:{serverId}:][weapon:{weaponId}:]players:{playerId}`
* @param player weapon to generate the report for
* @param server optional server filter
* @param weapon optional player filter
*/
export async function processPlayerReport(
player: string,
server?: number,
weapon?: string
) {
if (player == 'last_entry' || player == 'processedList') {
console.error('Player cannot be ' + player)
return
}
const serverPrefix = server ? `servers:${server}:` : ''
const weaponPrefix = weapon ? `weapons:${weapon}:` : ''
const cacheLocation = serverPrefix + weaponPrefix + `players:` + player
const last_entry = Number(await cache.HGET(cacheLocation, 'last_entry')) || 0
let query = db
.selectFrom('kill')
.select([
count<number>('id')
.filterWhere('attacker_id', '=', player)
.filterWhereRef('attacker_id', '!=', 'victim_id')
.as('kills'),
count<number>('id').filterWhere('victim_id', '=', player).as('deaths'),
avg<number>('kill.distance')
.filterWhere('attacker_id', '=', player)
.as('avg_kill_distance'),
max('kill.distance')
.filterWhere('attacker_id', '=', player)
.as('max_kill_distance'),
db
.selectFrom('kill')
.select(max('kill.id').as('last_entry'))
.as('last_entry')
])
.where((qb) =>
qb.where('attacker_id', '=', player).orWhere('victim_id', '=', player)
)
.where('kill.id', '>', last_entry)
if (server) {
query = query.where('server', '=', server)
}
if (weapon) {
query = query.where('cause_of_death', '=', weapon)
}
const newData = await query.execute()
if (newData.length == 0) {
return
}
const promises: Promise<any>[] = []
newData.forEach(({ kills, avg_kill_distance, max_kill_distance, deaths }) => {
promises.push(
processAvg(cacheLocation, 'avg_kill_distance', {
newkills: kills || 0,
newavg: avg_kill_distance || 0
}).then((e) => cache.HINCRBY(cacheLocation, 'kills', kills || 0)),
cache.HINCRBY(cacheLocation, 'deaths', deaths || 0),
processMax(cacheLocation, 'max_kill_distance', max_kill_distance)
)
})
promises.push(cache.HSET(cacheLocation, 'last_entry', newData[0].last_entry))
await Promise.all(promises)
}
async function processAvg(
cacheLocation: string,
key: 'avg_kill_distance',
{ newkills, newavg }: { newkills: number; newavg: number }
) {
const oldKills = Number(await cache.HGET(cacheLocation, 'kills')) || 0
const oldaverage = Number(await cache.HGET(cacheLocation, key)) || 0
const totalkills = Number(oldKills) + Number(newkills)
const newAverage =
(oldKills / totalkills) * oldaverage + (newkills / totalkills) * newavg || 0
await cache.HSET(cacheLocation, key, newAverage)
}
async function processMax(
cacheLocation: string,
key: 'max_kill_distance',
newmax: number
) {
const oldMax = Number(await cache.HGET(cacheLocation, key)) || 0
if (oldMax < newmax || oldMax == 0)
await cache.HSET(cacheLocation, key, newmax || 0)
}
/**
* Combines player reports in a readable json string.
*
* ProcessPlayerReport must be called manually before this function !
*
* Inputs `SET` `[servers:{serverId}:][weapons:{weaponId}:]players`
*
* Outputs `STRING` `[servers:{serverId}:][weapons:{weaponId}:]players:processedList`
* @param server optional server filter
* @param weapon optional player filter
* @returns
*/
export async function processPlayerList(server?: number, weapon?: string) {
const serverPrefix = server ? `servers:${server}:` : ''
const weaponPrefix = weapon ? `weapons:${weapon}:` : ''
const cacheLocation = serverPrefix + weaponPrefix + `players`
const players = await getPlayerSet(server, weapon)
const promises: Promise<any>[] = []
const data: {
[playerId: string]: {
[playerData: string]: string
}
} = {}
players.forEach((plr) => {
promises.push(
(async () => {
const playerData = await getPlayerReport(plr, server, weapon)
if (Object.keys(playerData).length > 0) {
data[plr] = playerData
}
})()
)
})
await Promise.all(promises)
return await cache.SET(cacheLocation + `:processedList`, JSON.stringify(data))
}
+14 -12
View File
@@ -16,7 +16,7 @@ const { count, max, avg } = db.fn
* @param player optional player filter
* @returns
*/
export async function populateWeaponSet(server?: number, player?: number) {
export async function populateWeaponSet(server?: number, player?: string) {
const serverPrefix = server ? `servers:${server}:` : ''
const playerPrefix = player ? `players:${player}:` : ''
const cacheLocation = serverPrefix + playerPrefix + `weapons`
@@ -37,8 +37,8 @@ export async function populateWeaponSet(server?: number, player?: number) {
}
if (player) {
query = query
.where('attacker_id', '=', player.toString())
.orWhere('victim_id', '=', player.toString())
.where('attacker_id', '=', player)
.orWhere('victim_id', '=', player)
}
const newData = await query.execute()
if (newData.length == 0) {
@@ -71,7 +71,7 @@ export async function populateWeaponSet(server?: number, player?: number) {
export async function processWeaponReport(
weapon: string,
server?: number,
player?: number
player?: string
) {
if (weapon == 'last_entry' || weapon == 'processedList') {
console.error('Weapon cannot be ' + weapon)
@@ -93,15 +93,17 @@ export async function processWeaponReport(
.as('last_entry')
])
.where('cause_of_death', '=', weapon)
.whereRef('attacker_id', '!=', 'victim_id')
.where('kill.id', '>', last_entry)
.groupBy('cause_of_death')
if (server) {
query = query.where('server', '=', server)
}
if (player) {
query = query.where('attacker_id', '=', player.toString())
query = query.where('attacker_id', '=', player)
}
const newData = await query.execute()
if (newData.length == 0) {
return
}
@@ -137,7 +139,9 @@ async function processMax(
key: 'max_kill_distance',
newmax: number
) {
const oldMax = Number(await cache.HGET(cacheLocation, key)) | 0
const oldMax = Number(await cache.HGET(cacheLocation, key)) || -1
newmax = newmax || 0
if (!oldMax || !newmax) console.log(oldMax, newmax)
if (oldMax < newmax) await cache.HSET(cacheLocation, key, newmax)
}
@@ -153,10 +157,11 @@ async function processMax(
* @param player optional player filter
* @returns
*/
export async function processWeaponList(server?: number, player?: number) {
export async function processWeaponList(server?: number, player?: string) {
const serverPrefix = server ? `servers:${server}:` : ''
const playerPrefix = player ? `players:${player}:` : ''
const weapons = await cache.SMEMBERS('weapons')
const cacheLocation = serverPrefix + playerPrefix + `weapons`
const weapons = await cache.SMEMBERS(cacheLocation)
const promises: Promise<any>[] = []
const data: {
[weaponId: string]: {
@@ -174,8 +179,5 @@ export async function processWeaponList(server?: number, player?: number) {
)
})
await Promise.all(promises)
return await cache.SET(
serverPrefix + playerPrefix + `weapons:processedList`,
JSON.stringify(data)
)
return await cache.SET(cacheLocation + `:processedList`, JSON.stringify(data))
}
+11 -12
View File
@@ -2,7 +2,8 @@ import { RequestHandler } from 'express'
import { param } from 'express-validator'
import { validateErrors } from '../../common'
import cache from '../../cache/redis'
import { sql } from 'kysely'
import { getWeaponList, getPlayerReport } from '../../cache/cacheUtils'
import { DataTypeNode, sql } from 'kysely'
import db from '../../db/db'
//import { getPlayerWeapons } from './serverPlayerWeapons'
@@ -13,22 +14,20 @@ const middlewares: RequestHandler[] = [
param(['serverId', 'playerId']).exists().toInt().isInt(),
validateErrors,
async (req, res) => {
const server = Number(req.params.serverId)
const player = Number(req.params.playerId)
//await processPlayerReport(server, player)
const data = await getPlayerReport(server, player)
//const weapons = await getPlayerWeapons(server, player)
const data = await getPlayerReport(
req.params.playerId,
Number(req.params.serverId)
)
const weapons = await getWeaponList(
req.params.serverId,
req.params.playerId
)
data.weapons = weapons
res.status(200).send(data)
//res.status(200).send({ ...data, weapons })
}
]
async function getPlayerReport(server: number, player: number) {
const data = await cache.HGETALL(`servers:${server}:players:${player}`)
delete data.last_entry
return data
}
export async function processPlayerReport(server: number, player: number) {
let last_entry =
Number(
+2 -1
View File
@@ -10,7 +10,8 @@ const db = new Kysely<Database>({
host: process.env.POSTGRES_HOST,
database: process.env.POSTGRES_DATABASE,
user: process.env.POSTGRES_USER,
password: process.env.POSTGRES_PASSWORD
password: process.env.POSTGRES_PASSWORD,
max: 30
})
}),
log(event) {