Performance update

This commit is contained in:
2023-05-30 19:31:04 +02:00
parent 1b78aecb13
commit 886384b3bf
11 changed files with 380 additions and 297 deletions
+1
View File
@@ -3,3 +3,4 @@ POSTGRES_DATABASE="postgres"
POSTGRES_USER="postgres"
POSTGRES_PASSWORD="postgres"
SERVERAUTH_TOKEN="1:f9f8fc8c-9185-47a8-8a99-9e3acf6ca007"
ENVIRONMENT='prod'
+4 -1
View File
@@ -10,5 +10,8 @@ module.exports = {
sourceType: 'module',
project: ['./tsconfig.json']
},
rules: {}
rules: {
'@typescript-eslint/strict-boolean-expressions': 0,
'@typescript-eslint/explicit-function-return-type': 'off'
}
}
+5 -6
View File
@@ -1,11 +1,10 @@
{
"editor.codeActionsOnSave": {
"source.fixAll": true
},
"editor.formatOnSave": true,
"typescript.preferences.quoteStyle": "single",
"javascript.preferences.quoteStyle": "single",
"prettier.singleQuote": true,
"prettier.trailingComma": "none",
"prettier.semi": false
"editor.defaultFormatter": "dbaeumer.vscode-eslint",
"eslint.format.enable": true,
"[typescript]": {
"editor.defaultFormatter": "dbaeumer.vscode-eslint"
}
}
+99 -130
View File
@@ -1,149 +1,118 @@
import { Router } from 'express'
import { param, query } from 'express-validator'
import { param } from 'express-validator'
import { validateErrors } from '../common'
import { allData } from '../process/process'
import { getHostList } from '../db/db'
import db, { getHostList } from '../db/db'
import { sql } from 'kysely'
const { max, sum } = db.fn
const router = Router()
// timeout middleware ?
router.get('/*', (req, res, next) => {
next()
})
function computeQueryParam(
queryParam: string,
comparison: string | number
): boolean {
return queryParam.startsWith('!')
? queryParam.substring(1) != comparison
: queryParam == comparison
}
const queryFilters = {
player: 'attacker_id',
server: 'servername',
map: 'map',
weapon: 'cause_of_death',
gamemode: 'game_mode',
host: 'host'
} as const
function filters(e: typeof allData[0], query: any) {
return (
(query.player ? computeQueryParam(query.player, e.attacker_id) : true) &&
(query.server ? computeQueryParam(query.server, e.servername) : true) &&
(query.host ? computeQueryParam(query.host, e.host) : true) &&
(query.map ? computeQueryParam(query.map, e.map) : true) &&
(query.weapon ? computeQueryParam(query.weapon, e.cause_of_death) : true) &&
(query.gamemode ? computeQueryParam(query.gamemode, e.game_mode) : true)
)
}
const path = {
player: 'attacker_id',
servers: 'servername',
maps: 'map',
weapons: 'cause_of_death',
gamemodes: 'game_mode',
hosts: 'host'
} as const
router.get('/hosts', async (req, res) => {
const result = await getHostList()
const data: { [key: number]: string } = {}
result.forEach((e) => (data[Number(e.id)] = e.name))
res.status(200).send(data)
})
router.get(
'/:dataType',
param('dataType')
.custom(
(e) =>
e == 'weapons' ||
e == 'players' ||
e == 'maps' ||
e == 'servers' ||
e == 'gamemodes'
)
.withMessage('Only weapons, players, maps or servers are valid paths'),
query(['server', 'map', 'weapon', 'gamemode', 'player', 'host'])
.optional()
.isString(),
validateErrors,
(req, res) => {
const data: {
[key: string]: {
deaths: number
interface KillRecord {
kills: number
deaths?: number
deaths_while_equipped?: number
username?: string
max_distance: number
total_distance: number
username?: string
host?: number
deaths_while_equipped?: number
}
} = {}
let index:
| 'cause_of_death'
| 'attacker_id'
| 'map'
| 'servername'
| 'game_mode'
switch (req.params.dataType) {
case 'weapons':
index = 'cause_of_death'
break
case 'players':
index = 'attacker_id'
break
case 'maps':
index = 'map'
break
case 'servers':
index = 'servername'
break
case 'gamemodes':
index = 'game_mode'
break
default:
return res.status(400).send()
}
const timeStart = new Date()
allData
.filter((e) => filters(e, req.query))
.forEach((e) => {
if (
!e.cause_of_death ||
!e.attacker_id ||
!e.map ||
!e.servername ||
!e.game_mode
)
return
const requestIndex = e[index]
if (!requestIndex) return
if (!data[requestIndex])
data[requestIndex] = {
deaths: 0,
kills: 0,
max_distance: 0,
total_distance: 0
}
if (index === 'attacker_id')
data[requestIndex].username = e.attacker_name
if (index === 'servername') data[requestIndex].host = e.host
if (
index === 'cause_of_death' ||
(req.query.weapon &&
!req.query.weapon.toString().startsWith('!') &&
index === 'attacker_id')
)
data[requestIndex].deaths_while_equipped =
Number(e.deaths_with_weapon) +
(data[requestIndex].deaths_while_equipped || 0)
data[requestIndex].deaths += Number(e.deaths)
data[requestIndex].kills += Number(e.kills)
data[requestIndex].total_distance += Number(e.total_distance)
data[requestIndex].max_distance = Math.max(
data[requestIndex].max_distance,
Number(e.max_distance)
)
router.get('/hosts', (_req, res) => {
void (async () => {
const result = await getHostList()
const data: Record<number, string> = {}
result.forEach((e) => (data[Number(e.id)] = e.name))
res.status(200).send(data)
})()
})
console.log(
new Date().toLocaleString() + ',' +
(req.headers['x-forwarded-for']?.toString() ||
req.socket.remoteAddress?.toString() ||
'') +
',' +
Math.abs(new Date().getTime() - timeStart.getTime()) / 1000 + ',' + req.originalUrl
)
const dataString = JSON.stringify(data)
const buffer = Buffer.from(dataString)
const size = buffer.length
res.status(200).setHeader('X-File-Size', size).setHeader('Content-Type', 'application/json').send(buffer)
function processQueryArgs (data: ReturnType<typeof db.selectFrom<'kill_view'>>, queryArgs: string | string[]): ReturnType<typeof db.selectFrom<'kill_view'>> {
Object.entries(queryArgs).forEach(([one, two]) => {
if (!two) return
if (!(one in queryFilters)) return
const key = one as keyof typeof queryFilters
if (Array.isArray(two)) {
data = data.where((qb) => {
(two as string[]).forEach((e) => { qb = qb.orWhere(queryFilters[key], '=', e) })
return qb
})
} else {
data = data.where(queryFilters[key], '=', two)
}
)
})
return data
}
router.get('/players',
(req, res) => {
void (async () => {
let data = db.selectFrom('kill_view')
if (req.query) {
data = processQueryArgs(data, req.query as unknown as string | string[])
}
const selection = data.select([sum<number>('kills').as('kills'), sum<number>('deaths').as('deaths'), sum<number>('deaths_with_weapon').as('deaths_while_equipped'), 'attacker_id', sql<string>`last(attacker_name)`.as('username'), sum<number>('total_distance').as('total_distance'), max('max_distance').as('max_distance')]).groupBy('attacker_id')
const test = (await selection.execute()).reduce<Record<string, KillRecord>>((acc, curr) => {
acc[curr.attacker_id] = {
kills: Number(curr.kills),
deaths: Number(curr.deaths),
max_distance: Number(curr.max_distance),
total_distance: Number(curr.total_distance),
deaths_while_equipped: Number(curr.deaths_while_equipped),
username: curr.username
}
return acc
}, {})
res.send(test)
})()
})
router.get('/:dataType',
param('dataType')
.custom((e) => e in path)
.withMessage('Only weapons, players, maps, gamemodes or servers are valid paths'),
validateErrors,
(req, res) => {
void (async () => {
let data = db.selectFrom('kill_view')
if (req.query) {
data = processQueryArgs(data, req.query as unknown as string | string[])
}
const dataType = req.params.dataType as keyof typeof path
const selection = data.select([sum<number>('kills').as('kills'), sum<number>('deaths').as('deaths'), sum<number>('deaths_with_weapon').as('deaths_while_equipped'), path[dataType], sum<number>('total_distance').as('total_distance'), max('max_distance').as('max_distance')]).groupBy(path[dataType])
const test = (await selection.execute()).reduce<Record<string, KillRecord>>((acc, curr) => {
acc[curr[path[dataType]]] = {
kills: Number(curr.kills),
deaths: Number(curr.deaths),
max_distance: Number(curr.max_distance),
total_distance: Number(curr.total_distance),
deaths_while_equipped: Number(curr.deaths_while_equipped)
}
return acc
}, {})
res.send(test)
})()
})
export default router
+12 -16
View File
@@ -1,36 +1,33 @@
import * as dotenv from 'dotenv'
import cluster from 'cluster'
import os from 'os'
dotenv.config()
import express from 'express'
import cors from 'cors'
import client from './client/client'
import { dbReady } from './db/db'
import processAll from './process/process'
import listenKills from './process/onKill'
dotenv.config()
const cCPUs = os.cpus().length;
const cCPUs = os.cpus().length
const port = 3000
if (cluster.isPrimary) {
if (cluster.isPrimary && process.env.ENVIRONMENT !== 'dev') {
// Create a worker for each CPU
for (let i = 0; i < cCPUs; i++) {
cluster.fork();
cluster.fork()
}
cluster.on('online', function (worker) {
console.log('Worker ' + worker.process.pid + ' is online.');
});
console.log(`Worker ${worker.process.pid ?? ''} is online`)
})
cluster.on('exit', function (worker, code, signal) {
console.log('worker ' + worker.process.pid + ' died.');
});
console.log(`Worker ${worker.process.pid ?? ''} died`)
})
}
export default
new Promise(async (resolve, reject) => {
if (!cluster.isPrimary) {
new Promise((resolve, reject) => {
void (async () => {
if (!cluster.isPrimary || process.env.ENVIRONMENT === 'dev') {
const app = express()
app.use(express.json())
app.use(cors({ exposedHeaders: ['X-File-Size', 'Content-Encoding'] }))
@@ -41,11 +38,10 @@ export default
app.use('/', client)
await dbReady()
await processAll()
await listenKills()
const listenServer = app.listen(port, '0.0.0.0', () => {
console.log(`Tone client api listening on port ${port}`)
resolve(listenServer)
})
}
})()
})
+5 -5
View File
@@ -1,11 +1,11 @@
import { RequestHandler } from 'express'
import { Result, validationResult } from 'express-validator'
import { type RequestHandler } from 'express'
import { validationResult } from 'express-validator'
import http from 'http'
import https from 'https'
export function GetRequest(url: string) {
return new Promise<string>((resolve, reject) => {
let handler = (resp: http.IncomingMessage) => {
export async function GetRequest (url: string) {
return await new Promise<string>((resolve, reject) => {
const handler = (resp: http.IncomingMessage) => {
let data = ''
// A chunk of data has been received.
+8 -7
View File
@@ -1,9 +1,10 @@
import * as dotenv from 'dotenv'
dotenv.config()
import { InsertObject, Kysely, PostgresDialect } from 'kysely'
import { Kysely, PostgresDialect } from 'kysely'
import { Pool } from 'pg'
import migrateToLatest from './migrations'
import Database, { KillTable } from './model'
import { type KillTable } from './model'
import type Database from './model'
dotenv.config()
const migration = migrateToLatest()
const db = new Kysely<Database>({
dialect: new PostgresDialect({
@@ -16,7 +17,7 @@ const db = new Kysely<Database>({
})
}),
log (event) {
if (process.env.ENVIRONMENT != 'production') {
if (process.env.ENVIRONMENT !== 'dev') {
return
}
if (event.level === 'query') {
@@ -40,12 +41,12 @@ export async function CreateKillRecord(data: KillRecord) {
.execute()
}
export async function getHostList () {
return await db.selectFrom("host").select(['id', 'host.name']).execute()
return await db.selectFrom('host').select(['id', 'host.name']).execute()
}
// tokens are stored in raw... maybe we should use something better in the future
// Using callback for express-basic-auth
export function CheckServerToken(token: string) {
return db.selectFrom('host').select('id')
export async function CheckServerToken (token: string) {
return await db.selectFrom('host').select('id')
.where('token', '=', Buffer.from(token, 'base64').toString())
.executeTakeFirst()
.then((result) => {
@@ -0,0 +1,126 @@
import { type Kysely } from 'kysely'
import { Client } from 'pg'
const pgClient = new Client({
host: process.env.POSTGRES_HOST,
database: process.env.POSTGRES_DATABASE,
user: process.env.POSTGRES_USER,
password: process.env.POSTGRES_PASSWORD
})
export async function up (db: Kysely<any>): Promise<void> {
await pgClient.connect()
await pgClient.query('DROP TABLE IF EXISTS kill_view;')
console.log('creating table kill_view')
await pgClient.query(`CREATE TABLE kill_view
AS (
with "kills" as (select count("id") as "kills", coalesce(max("distance"), 0) as "max_distance", coalesce(sum("distance"), 0) as "total_distance", "attacker_id", last(attacker_name) as "attacker_name", "cause_of_death", "map", "game_mode", "servername", "host" from "kill" where "attacker_id" != "victim_id" group by "attacker_id", "cause_of_death", "map", "servername", "host", "game_mode"), "deaths" as (select count("id") as "deaths", "victim_id", last(victim_name) as "victim_name", "cause_of_death", "map", "game_mode", "servername", "host" from "kill" group by "victim_id", "cause_of_death", "map", "servername", "host", "game_mode"), "deathswithweapon" as (select count("id") as "deaths", "victim_id", "victim_current_weapon", "map", "game_mode", "servername", "host" from "kill" group by "victim_id", "victim_current_weapon", "map", "servername", "host", "game_mode") select COALESCE(kills.attacker_name, deaths.victim_name) as "attacker_name", COALESCE(kills.attacker_id, deaths.victim_id) as "attacker_id", COALESCE(kills.kills, 0) as "kills", COALESCE(kills.total_distance, 0) as "total_distance", COALESCE(deathswithweapon.deaths, 0) as "deaths_with_weapon", COALESCE(kills.max_distance, 0) as "max_distance", COALESCE(deaths.deaths, 0) as "deaths", COALESCE(kills.servername, deaths.servername) as "servername", COALESCE(kills.host, deaths.host) as "host", COALESCE(kills.cause_of_death, deaths.cause_of_death) as "cause_of_death", COALESCE(kills.map, deaths.map) as "map", COALESCE(kills.game_mode, deaths.game_mode) as "game_mode" from "kills" full join "deaths" on "deaths"."victim_id" = "kills"."attacker_id" and "deaths"."cause_of_death" = "kills"."cause_of_death" and "deaths"."servername" = "kills"."servername" and "deaths"."host" = "kills"."host" and "deaths"."game_mode" = "kills"."game_mode" and "deaths"."map" = "kills"."map" left join "deathswithweapon" on "deathswithweapon"."victim_id" = "kills"."attacker_id" and "deathswithweapon"."victim_current_weapon" = "kills"."cause_of_death" and "deathswithweapon"."servername" = "kills"."servername" and "deathswithweapon"."host" = "kills"."host" and "deathswithweapon"."game_mode" = "kills"."game_mode" and "deathswithweapon"."map" = "kills"."map"
);`)
console.log('creating index on attacker_id')
await pgClient.query('CREATE INDEX ON kill_view USING HASH ("attacker_id");')
console.log('creating index on map')
await pgClient.query('CREATE INDEX ON kill_view USING HASH ("map");')
console.log('creating index on game_mode')
await pgClient.query('CREATE INDEX ON kill_view USING HASH ("game_mode");')
console.log('creating index on cause_of_death')
await pgClient.query('CREATE INDEX ON kill_view USING HASH ("cause_of_death");')
console.log('creating index on server')
await pgClient.query('CREATE INDEX ON kill_view ("servername", "host");')
console.log('creating function update_kill_view_fnct')
await pgClient.query(`CREATE OR REPLACE FUNCTION update_kill_view_fnct()
RETURNS TRIGGER
AS $$
BEGIN
IF EXISTS(
SELECT * FROM kill_view
WHERE
new.attacker_id = attacker_id AND
new.map = map AND
new.game_mode = game_mode AND
new.cause_of_death = cause_of_death AND
new.servername = servername AND
new.host = host
) THEN
UPDATE kill_view SET kills = kills + 1,
max_distance = (SELECT Max(v) FROM (VALUES (new.distance), (max_distance)) AS value(v)),
total_distance = total_distance + new.distance,
attacker_name = new.attacker_name
WHERE
new.attacker_id = attacker_id AND
new.map = map AND
new.game_mode = game_mode AND
new.cause_of_death = cause_of_death AND
new.servername = servername AND
new.host = host;
ELSE
INSERT INTO kill_view(kills,deaths, deaths_with_weapon, attacker_id, map, game_mode, cause_of_death, servername, host, max_distance, total_distance)
VALUES(1,0,0, new.attacker_id, new.map, new.game_mode, new.cause_of_death, new.servername, new.host,new.distance, new.distance);
END IF;
IF EXISTS(
SELECT * FROM kill_view
WHERE
new.victim_id = attacker_id AND
new.map = map AND
new.game_mode = game_mode AND
new.cause_of_death = cause_of_death AND
new.servername = servername AND
new.host = host
) THEN
UPDATE kill_view SET deaths = deaths + 1
WHERE
new.victim_id = attacker_id AND
new.map = map AND
new.game_mode = game_mode AND
new.cause_of_death = cause_of_death AND
new.servername = servername AND
new.host = host;
ELSE
INSERT INTO kill_view(deaths, kills, deaths_with_weapon, attacker_id, map, game_mode, cause_of_death, servername, host, max_distance, total_distance)
VALUES(1,0,0, new.victim_id, new.map, new.game_mode, new.cause_of_death, new.servername, new.host,0,0);
END IF;
IF EXISTS(
SELECT * FROM kill_view
WHERE
new.victim_id = attacker_id AND
new.map = map AND
new.game_mode = game_mode AND
new.attacker_current_weapon = cause_of_death AND
new.servername = servername AND
new.host = host
) THEN
UPDATE kill_view SET deaths_with_weapon = deaths_with_weapon + 1
WHERE
new.victim_id = attacker_id AND
new.map = map AND
new.game_mode = game_mode AND
new.attacker_current_weapon = cause_of_death AND
new.servername = servername AND
new.host = host;
ELSE
INSERT INTO kill_view(deaths, kills, deaths_with_weapon, attacker_id, map, game_mode, cause_of_death, servername, host, max_distance, total_distance)
VALUES(0,0,1, new.victim_id, new.map, new.game_mode, new.attacker_current_weapon, new.servername, new.host,0,0);
END IF;
RETURN NEW;
END;
$$ LANGUAGE PLPGSQL;`)
console.log('creating trigger update_kill_view')
await pgClient.query(`CREATE OR REPLACE TRIGGER update_kill_view
AFTER INSERT ON kill
FOR EACH ROW
EXECUTE FUNCTION update_kill_view_fnct();`)
await pgClient.end()
}
export async function down (db: Kysely<any>): Promise<void> {
await pgClient.connect()
await pgClient.query('DROP TRIGGER IF EXISTS update_kill_view;')
await pgClient.query('DROP FUNCTION IF EXISTS update_kill_view_fnct')
await pgClient.query('DROP TABLE IF EXISTS kill_view;')
await pgClient.end()
}
+16 -23
View File
@@ -1,4 +1,4 @@
import { ColumnType, Generated } from 'kysely'
import { type Generated } from 'kysely'
export interface KillTable {
id: Generated<number>
@@ -43,37 +43,30 @@ export interface KillTable {
distance: number
titan?: string
}
/*
interface PlayerTable {
id: number
name: string
'opt-out': boolean
hide_TOS: boolean
}
*/
interface WeaponTable {
id: string
name: string
description: string
image: string
}
interface MapTable {
id: string
name: string
description: string
image: string
export interface KillViewTable {
kills: number
deaths: number
deaths_with_weapon: number
attacker_id: number
attacker_name: string
map: string
game_mode: string
cause_of_death: string
servername: string
host: number
max_distance: number
total_distance: number
}
interface HosterTable {
id: Generated<number>
name: string
token: Generated<string>
}
interface Database {
kill_view: KillViewTable
kill: KillTable
//player: PlayerTable
weapon: WeaponTable
maps: MapTable
host: HosterTable
}
+5 -3
View File
@@ -1,5 +1,5 @@
import { Client } from 'pg'
import { allData } from './process'
// import { allData } from './process'
export const pgClient = new Client({
host: process.env.POSTGRES_HOST,
@@ -12,9 +12,10 @@ export default async function listenKills() {
await pgClient.connect()
await pgClient.query('LISTEN new_kill')
pgClient.on('notification', (data) => {
/* pgClient.on('notification', (data) => {
if (!data.payload) return
const payload = JSON.parse(data.payload)
let killEntry = allData.find(
(e) =>
e.attacker_id == payload.attacker_id &&
@@ -106,6 +107,7 @@ export default async function listenKills() {
deathWithWeaponEntry.deaths_with_weapon++
deathWithWeaponEntry.attacker_name = payload.victim_name
killEntry.attacker_name = payload.attacker_name
})
}) */
return pgClient
}
+5 -12
View File
@@ -1,6 +1,6 @@
import db from '../db/db'
const { count, max, sum, coalesce } = db.fn
import { sql } from 'kysely'
const { count, max, sum, coalesce } = db.fn
export let allData: Awaited<ReturnType<typeof processGlobalStats>>
@@ -8,20 +8,13 @@ export let allData: Awaited<ReturnType<typeof processGlobalStats>>
* populates allData
* @returns
*/
async function processAll() {
async function processAll (): Promise<void> {
console.log('Starting data calculation...')
const timeStart = new Date()
allData = await processGlobalStats()
// allData = await processGlobalStats()
console.log(
'Data calculation finished. Took + ' +
Math.abs(new Date().getTime() - timeStart.getTime()) / 1000 +
' seconds'
)
if (process.env.ENVIRONMENT == 'production') {
return
}
console.log(`Data calculation finished. Took + ${Math.abs(new Date().getTime() - timeStart.getTime()) / 1000} seconds`)
}
async function processGlobalStats () {
@@ -130,7 +123,7 @@ async function processGlobalStats() {
sql<number>`COALESCE(kills.host, deaths.host)`.as('host'),
sql<string>`COALESCE(kills.cause_of_death, deaths.cause_of_death)`.as('cause_of_death'),
sql<string>`COALESCE(kills.map, deaths.map)`.as('map'),
sql<string>`COALESCE(kills.game_mode, deaths.game_mode)`.as('game_mode'),
sql<string>`COALESCE(kills.game_mode, deaths.game_mode)`.as('game_mode')
])
.execute()
}