From a297c868302ef3449b4f59ba41b0c7847d8ad8f3 Mon Sep 17 00:00:00 2001 From: legonzaur Date: Sun, 9 Apr 2023 14:47:56 +0200 Subject: [PATCH] process rework, some tests and instant update --- src/client/process.ts | 227 ------------------ src/client/process/processPlayer.ts | 42 ---- src/client/process/processWeapon.ts | 42 ---- .../migrations/2023-04-09-01 kill watcher.ts | 34 +++ src/process/onKill.ts | 71 ++++++ src/process/process.ts | 184 ++++++++++++++ src/processMain.ts | 4 +- tests/client.test.ts | 36 +++ tests/realtime.test.ts | 110 +++++++++ 9 files changed, 438 insertions(+), 312 deletions(-) delete mode 100644 src/client/process.ts delete mode 100644 src/client/process/processPlayer.ts delete mode 100644 src/client/process/processWeapon.ts create mode 100644 src/db/migrations/2023-04-09-01 kill watcher.ts create mode 100644 src/process/onKill.ts create mode 100644 src/process/process.ts create mode 100644 tests/realtime.test.ts diff --git a/src/client/process.ts b/src/client/process.ts deleted file mode 100644 index 6df4ecc..0000000 --- a/src/client/process.ts +++ /dev/null @@ -1,227 +0,0 @@ -import db from '../db/db' -const { count, max, sum } = db.fn -import client from '../cache/redis' -import { createWeaponJson } from './process/processWeapon' -import { createPlayerJson } from './process/processPlayer' - -/** - * Starts the process of populating the REDIS database globally - * @returns - */ -async function processAll() { - console.log('Starting data calculation...') - const timeStart = new Date() - - await Promise.all([processGlobalStats(), processServerStats()]) - console.log( - 'Data calculation finished. Took + ' + - Math.abs(new Date().getTime() - timeStart.getTime()) / 1000 + - ' seconds' - ) - if (process.env.ENVIRONMENT == 'production') { - return - } - setTimeout(processAll, 3600000) -} - -async function processGlobalStats() { - let promises: Promise[] = [] - //Global kills - promises.push(db.selectFrom('kill') - .select([count('id').as('kills'), max('distance').as('max_distance'), sum('distance').as('total_distance')]) - .execute().then(data => { - let transaction = client.multi() - data.forEach(({ kills, max_distance, total_distance }) => { - transaction = processGlobalData({ kills, max_distance, total_distance }, transaction) - }) - return transaction.exec() - })) - - //Global weapon kill - promises.push(db.selectFrom('kill') - .select([count('id').as('kills'), max('distance').as('max_distance'), sum('distance').as('total_distance'), 'cause_of_death']) - .groupBy('cause_of_death').execute().then(async (data) => { - let transaction = client.multi() - data.forEach(({ kills, max_distance, total_distance, cause_of_death }) => { - transaction = processWeaponData({ kills, max_distance, total_distance, cause_of_death }, transaction) - }) - await transaction.exec() - })) - - //Player weapon kills - promises.push(db.selectFrom('kill') - .select([count('id').as('kills'), max('distance').as('max_distance'), sum('distance').as('total_distance'), 'cause_of_death', 'attacker_id']) - .groupBy(['cause_of_death', 'attacker_id']).execute().then(async (data) => { - let transaction = client.multi() - data.forEach(({ kills, max_distance, total_distance, cause_of_death, attacker_id }) => { - transaction = processWeaponData({ kills, max_distance, total_distance, cause_of_death, attacker_id }, transaction) - }) - await transaction.exec() - })) - //Player weapon deaths - promises.push(db.selectFrom('kill').select([count('id').as('deaths'), 'victim_id', 'cause_of_death']) - .groupBy(['victim_id', 'cause_of_death']).execute().then(data => { - let transaction = client.multi() - data.forEach(({ deaths, victim_id, cause_of_death }) => { - transaction = transaction.HSET(`weapons:${cause_of_death}:players:${victim_id}`, "deaths", deaths.toString()) - }) - return transaction.exec() - })) - - //Global player kills - promises.push(db.selectFrom('kill') - .select([count('id').as('kills'), max('distance').as('max_distance'), sum('distance').as('total_distance'), 'attacker_id']) - .groupBy('attacker_id').execute().then(data => { - let transaction = client.multi() - data.forEach(({ kills, max_distance, total_distance, attacker_id }) => { - processPlayerData({ kills, max_distance, total_distance, attacker_id }, transaction) - }) - return transaction.exec() - }) - ) - //Global player deaths - promises.push(db.selectFrom('kill').select([count('id').as('deaths'), 'victim_id']) - .groupBy('victim_id').execute().then(data => { - let transaction = client.multi() - data.forEach(({ deaths, victim_id }) => { - transaction = transaction.HSET('players:' + victim_id, "deaths", deaths.toString()) - }) - return transaction.exec() - })) - - await Promise.all(promises) - promises = []; - promises.push(createWeaponJson()); - promises.push(createPlayerJson()); - (await client.sMembers('players')).forEach((player: string) => { - promises.push(createWeaponJson(undefined, player)) - }); - (await client.sMembers('weapons')).forEach((weapon: string) => { - promises.push(createPlayerJson(undefined, weapon)) - }); - await Promise.all(promises) -} - -async function processServerStats() { - let promises: Promise[] = [] - //Server global kills - promises.push(db.selectFrom('kill') - .select([count('id').as('kills'), max('distance').as('max_distance'), sum('distance').as('total_distance'), 'server']) - .groupBy(['server']).execute().then(data => { - let transaction = client.multi() - data.forEach(({ kills, max_distance, total_distance, server }) => { - transaction = processGlobalData({ kills, max_distance, total_distance, server }, transaction) - }) - return transaction.exec() - })) - //Server weapon kills - promises.push(db.selectFrom('kill') - .select([count('id').as('kills'), max('distance').as('max_distance'), sum('distance').as('total_distance'), 'cause_of_death', 'server']) - .groupBy(['cause_of_death', 'server']).execute().then(data => { - let transaction = client.multi() - data.forEach(({ kills, max_distance, total_distance, cause_of_death, server }) => { - transaction = processWeaponData({ kills, max_distance, total_distance, cause_of_death, server }, transaction) - }) - return transaction.exec() - })) - - //Server Player weapon kills - promises.push(db.selectFrom('kill') - .select([count('id').as('kills'), max('distance').as('max_distance'), sum('distance').as('total_distance'), 'cause_of_death', 'attacker_id', 'server']) - .groupBy(['cause_of_death', 'attacker_id', 'server']).execute().then(data => { - let transaction = client.multi() - data.forEach(({ kills, max_distance, total_distance, cause_of_death, attacker_id, server }) => { - transaction = processWeaponData({ kills, max_distance, total_distance, cause_of_death, attacker_id, server }, transaction) - }) - return transaction.exec() - })) - //Server Player weapon deaths - promises.push(db.selectFrom('kill').select([count('id').as('deaths'), 'victim_id', 'cause_of_death', 'server']) - .groupBy(['victim_id', 'cause_of_death', 'server']).execute().then(data => { - let transaction = client.multi() - data.forEach(({ deaths, victim_id, cause_of_death, server }) => { - transaction = transaction.HSET(`servers:${server}:weapons:${cause_of_death}:players:${victim_id}`, "deaths", deaths.toString()) - }) - return transaction.exec() - })) - - //server player kills - promises.push(db.selectFrom('kill') - .select([count('id').as('kills'), max('distance').as('max_distance'), sum('distance').as('total_distance'), 'attacker_id', 'server']) - .groupBy(['attacker_id', 'server']).execute().then(data => { - let transaction = client.multi() - data.forEach(({ kills, max_distance, total_distance, attacker_id, server }) => { - transaction = processPlayerData({ kills, max_distance, total_distance, attacker_id, server }, transaction) - }) - return transaction.exec() - }) - ) - - //server player deaths - promises.push(db.selectFrom('kill').select([count('id').as('deaths'), 'victim_id', 'server']) - .groupBy(['victim_id', 'server']).execute().then(data => { - let transaction = client.multi() - data.forEach(({ deaths, victim_id, server }) => { - transaction = transaction.HSET('servers:' + server + ':players:' + victim_id, "deaths", deaths.toString()) - }) - return transaction.exec() - })) - - await Promise.all(promises) - promises = []; - ((await client.sMembers('servers')).forEach((server: string) => { - if (isNaN(Number(server))) return - promises.push(createWeaponJson(Number(server))); - promises.push(createPlayerJson(Number(server))); - promises.push(client.sMembers('servers:' + server + ':players').then(data => { - data.forEach((player: string) => { - promises.push(createWeaponJson(Number(server), player)) - }) - })) - promises.push(client.sMembers('servers:' + server + 'weapons').then(data => { - data.forEach((weapon: string) => { - promises.push(createPlayerJson(Number(server), weapon)) - }) - })) - })); - - await Promise.all(promises) -} - -function processGlobalData({ kills, max_distance, total_distance, server }: { kills: string | number | bigint, max_distance: number, total_distance: string | number | bigint, server?: number }, transaction: any) { - const serverPrefix = server ? `servers:${server}:` : '' - const cacheLocation = serverPrefix + 'global' - transaction = transaction.HSET(cacheLocation, "total_distance", total_distance.toString()) - transaction = transaction.HSET(cacheLocation, "max_distance", max_distance.toString()) - transaction = transaction.HSET(cacheLocation, "kills", kills.toString()) - if (server) { - transaction = transaction.SADD('servers', server.toString()) - } - - return transaction -} - -function processWeaponData({ kills, max_distance, total_distance, cause_of_death, attacker_id, server }: { kills: string | number | bigint, max_distance: number, total_distance: string | number | bigint, cause_of_death: string, attacker_id?: string, server?: number }, transaction: any) { - const serverPrefix = server ? `servers:${server}:` : '' - const playerPrefix = attacker_id ? `players:${attacker_id}:` : '' - const cacheLocation = serverPrefix + playerPrefix + `weapons:` + cause_of_death - transaction = transaction.HSET(cacheLocation, "total_distance", total_distance.toString()) - transaction = transaction.HSET(cacheLocation, "max_distance", max_distance.toString()) - transaction = transaction.HSET(cacheLocation, "kills", kills.toString()) - transaction = transaction.SADD(serverPrefix + playerPrefix + `weapons`, cause_of_death.toString()) - return transaction -} - -function processPlayerData({ kills, max_distance, total_distance, attacker_id, cause_of_death, server }: { kills: string | number | bigint, max_distance: number, total_distance: string | number | bigint, attacker_id: string, server?: number, cause_of_death?: string }, transaction: any) { - const serverPrefix = server ? `servers:${server}:` : '' - const weaponPrefix = cause_of_death ? `weapons:${cause_of_death}:` : '' - const cacheLocation = serverPrefix + weaponPrefix + `players:` + attacker_id - - transaction = transaction.HSET(cacheLocation, "total_distance", total_distance.toString()) - transaction = transaction.HSET(cacheLocation, "max_distance", max_distance.toString()) - transaction = transaction.HSET(cacheLocation, "kills", kills.toString()) - transaction = transaction.SADD(serverPrefix + weaponPrefix + `players`, attacker_id.toString()) - return transaction -} - -export default processAll diff --git a/src/client/process/processPlayer.ts b/src/client/process/processPlayer.ts deleted file mode 100644 index 4e8c92c..0000000 --- a/src/client/process/processPlayer.ts +++ /dev/null @@ -1,42 +0,0 @@ -/** - * This file contains all functions related to population of the REDIS database for players. - */ -import cache from '../../cache/redis' -import { getPlayerReport, getPlayerSet } from '../../cache/cacheUtils' - -/** - * Combines player reports in a readable json string. - * - * ProcessPlayerReport must be called manually before this function ! - * - * Inputs `SET` `[servers:{serverId}:][weapons:{weaponId}:]players` - * - * Outputs `STRING` `[servers:{serverId}:][weapons:{weaponId}:]players:processedList` - * @param server optional server filter - * @param weapon optional player filter - * @returns - */ -export async function createPlayerJson(server?: number, weapon?: string) { - const serverPrefix = server ? `servers:${server}:` : '' - const weaponPrefix = weapon ? `weapons:${weapon}:` : '' - const cacheLocation = serverPrefix + weaponPrefix + `players` - const players = await getPlayerSet(server, weapon) - const promises: Promise[] = [] - const data: { - [playerId: string]: { - [playerData: string]: string - } - } = {} - players.forEach((plr) => { - promises.push( - (async () => { - const playerData = await getPlayerReport(plr, server, weapon) - if (Object.keys(playerData).length > 0) { - data[plr] = playerData - } - })() - ) - }) - await Promise.all(promises) - return await cache.SET(cacheLocation + `:processedList`, JSON.stringify(data)) -} diff --git a/src/client/process/processWeapon.ts b/src/client/process/processWeapon.ts deleted file mode 100644 index 2cf87d8..0000000 --- a/src/client/process/processWeapon.ts +++ /dev/null @@ -1,42 +0,0 @@ -/** - * This file contains all functions related to population of the REDIS database for weapons. - */ -import cache from '../../cache/redis' -import { getWeaponReport } from '../../cache/cacheUtils' - -/** - * Combines weapons reports in a readable json string. - * - * ProcessWeaponReports must be called manually before this function ! - * - * Inputs `SET` `[servers:{serverId}:][players:{playerId}:]weapons` - * - * Outputs `STRING` `[servers:{serverId}:][players:{playerId}:]weapons:processedList` - * @param server optional server filter - * @param player optional player filter - * @returns - */ -export async function createWeaponJson(server?: number, player?: string) { - const serverPrefix = server ? `servers:${server}:` : '' - const playerPrefix = player ? `players:${player}:` : '' - const cacheLocation = serverPrefix + playerPrefix + `weapons` - const weapons = await cache.SMEMBERS(cacheLocation) - const promises: Promise[] = [] - const data: { - [weaponId: string]: { - [weaponData: string]: string - } - } = {} - weapons.forEach((wpn) => { - promises.push( - (async () => { - const weaponData = await getWeaponReport(wpn, server, player) - if (Object.keys(weaponData).length > 0) { - data[wpn] = weaponData - } - })() - ) - }) - await Promise.all(promises) - return await cache.SET(cacheLocation + `:processedList`, JSON.stringify(data)) -} diff --git a/src/db/migrations/2023-04-09-01 kill watcher.ts b/src/db/migrations/2023-04-09-01 kill watcher.ts new file mode 100644 index 0000000..c3438ac --- /dev/null +++ b/src/db/migrations/2023-04-09-01 kill watcher.ts @@ -0,0 +1,34 @@ +import { Kysely, sql } from 'kysely' +import { Client } from 'pg' + +const notify_newkill = ` +CREATE OR REPLACE FUNCTION notify_new_kill() + RETURNS trigger +AS +$$ +BEGIN + PERFORM pg_notify('new_kill', row_to_json(NEW)::text); + RETURN NULL; +END; +$$ +LANGUAGE plpgsql; +` + +const pgClient = new Client({ + host: process.env.POSTGRES_HOST, + database: process.env.POSTGRES_DATABASE, + user: process.env.POSTGRES_USER, + password: process.env.POSTGRES_PASSWORD +}) + +export async function up(db: Kysely): Promise { + await pgClient.connect() + await pgClient.query(notify_newkill) + await pgClient.query(`CREATE TRIGGER insert_kills_notify AFTER INSERT ON kill FOR EACH ROW EXECUTE PROCEDURE notify_new_kill();`) +} + +export async function down(db: Kysely): Promise { + await pgClient.query(`DROP FUNCTION IF EXISTS notify_new_kill;`) + await pgClient.query(`DROP TRIGGER IF EXISTS insert_kills_notify;`) + await pgClient.end() +} diff --git a/src/process/onKill.ts b/src/process/onKill.ts new file mode 100644 index 0000000..d847b56 --- /dev/null +++ b/src/process/onKill.ts @@ -0,0 +1,71 @@ +import { Client } from 'pg' +import client from '../cache/redis' + +const pgClient = new Client({ + host: process.env.POSTGRES_HOST, + database: process.env.POSTGRES_DATABASE, + user: process.env.POSTGRES_USER, + password: process.env.POSTGRES_PASSWORD +}) + +export default async function listenKills() { + await pgClient.connect() + await pgClient.query('LISTEN new_kill') + + pgClient.on('notification', async (data) => { + if (!data.payload) return + const payload = JSON.parse(data.payload) + console.log(JSON.parse(data.payload)) + + await updateGlobal(payload) + + }) +} + +async function updateGlobal({ cause_of_death, distance, attacker_id, victim_id, server }: { cause_of_death: string, distance: number, attacker_id: string, victim_id: string, server: number }) { + const promises: Promise[] = [] + + promises.push(updatePath(`data`, { distance })) + //servers.1 + promises.push(updatePath(`servers.${server}`, { distance }).then(() => Promise.all([ + //servers.1.players.123456789 + updatePath(`servers.${server}.players.${attacker_id}`, { distance }).then(() => + //servers.1.players.123456789.weapons.epg + updatePath(`servers.${server}.players.${attacker_id}.weapons.${cause_of_death}`, { distance }) + ), + //servers.1.weapons.epg + updatePath(`servers.${server}.weapons.${cause_of_death}`, { distance }).then(() => + //servers.1.weapons.epg.players.123456789 + updatePath(`servers.${server}.weapons.${cause_of_death}.players.${attacker_id}`, { distance }) + )]) + )) + + + //players.123456789 + promises.push(updatePath(`players.${attacker_id}`, { distance }).then(async () => { + await client.json.numIncrBy('kills', `players.${victim_id}.deaths`, 1) + //players.123456789.weapons.epg + await updatePath(`players.${attacker_id}.weapons.${cause_of_death}`, { distance }) + return await client.json.numIncrBy('kills', `server.${server}.players.${victim_id}.deaths`, 1) + })) + + + //weapons.epg + promises.push(updatePath(`weapons.${cause_of_death}`, { distance }).then(() => + //weapons.epg.players.123456789 + updatePath(`weapons.${cause_of_death}.players.${attacker_id}`, { distance }) + )) + + + await Promise.all(promises) +} + +async function updatePath(path: string, { distance }: { distance: number }) { + if (!await client.json.type('kills', path)) await client.json.set('kills', path, {}) + let transaction = client.multi() + transaction = transaction.json.numIncrBy('kills', `${path}.total_distance`, distance) + let max_distance = Number(await client.json.get('kills', { path: `${path}.max_distance` })) + if (isNaN(max_distance) || max_distance < distance) transaction = transaction.json.set('kills', `${path}.max_distance`, distance) + transaction = transaction.json.numIncrBy('kills', `${path}.kills`, 1) + return transaction.exec() +} \ No newline at end of file diff --git a/src/process/process.ts b/src/process/process.ts new file mode 100644 index 0000000..316fac4 --- /dev/null +++ b/src/process/process.ts @@ -0,0 +1,184 @@ +import db from '../db/db' +const { count, max, sum } = db.fn +import client from '../cache/redis' + +const genPrefix = { + global: ({ server }: { server?: number }) => { + return (server ? `servers.${server}.` : '') + 'data' + }, + weapon: ({ cause_of_death, server }: { cause_of_death: string, server?: number }) => { + return (server ? `servers.${server}.` : '') + `weapons.${cause_of_death}` + }, + player: ({ attacker_id, server }: { attacker_id: string, server?: number }) => { + return (server ? `servers.${server}.` : '') + `players.${attacker_id}` + }, + playerWeapons: ({ cause_of_death, attacker_id, server }: { attacker_id: string, cause_of_death: string, server?: number }) => { + return (server ? `servers.${server}.` : '') + `players.${attacker_id}.weapons.${cause_of_death}` + }, + weaponPlayers: ({ cause_of_death, attacker_id, server }: { attacker_id: string, cause_of_death: string, server?: number }) => { + return (server ? `servers.${server}.` : '') + `weapons.${cause_of_death}.players.${attacker_id}` + }, +} + +/** + * Starts the process of populating the REDIS database globally + * @returns + */ +async function processAll() { + console.log('Starting data calculation...') + const timeStart = new Date() + + if (!await client.json.type('kills')) await client.json.set('kills', '$', { data: {}, weapons: {}, players: {}, servers: {} }) + await Promise.all([processGlobalStats(), processServerStats()]) + console.log( + 'Data calculation finished. Took + ' + + Math.abs(new Date().getTime() - timeStart.getTime()) / 1000 + + ' seconds' + ) + if (process.env.ENVIRONMENT == 'production') { + return + } + setTimeout(processAll, 3600000) +} + +async function processGlobalStats() { + let promises: Promise[] = [] + //Global kills + await db.selectFrom('kill').select([count('id').as('kills'), max('distance').as('max_distance'), sum('distance').as('total_distance')]).execute() + .then(async (data) => { + let transaction = client.multi() + await Promise.all(data.map(async ({ kills, max_distance, total_distance }) => { + await processData(genPrefix.global({}), { total_distance, max_distance, kills }, transaction) + })) + return transaction.exec() + }) + + //Global weapon kill + await db.selectFrom('kill').select([count('id').as('kills'), max('distance').as('max_distance'), sum('distance').as('total_distance'), 'cause_of_death']).groupBy('cause_of_death').execute() + .then(async (data) => { + let transaction = client.multi() + await Promise.all(data.map(async ({ kills, max_distance, total_distance, cause_of_death }) => { + const prefix = genPrefix.weapon({ cause_of_death }) + await processData(prefix, { kills, max_distance, total_distance }, transaction) + })) + return transaction.exec() + }) + + //Global player kill + await db.selectFrom('kill').select([count('id').as('kills'), max('distance').as('max_distance'), sum('distance').as('total_distance'), 'attacker_id']).groupBy('attacker_id').whereRef("attacker_id", '!=', 'victim_id').execute() + .then(async (data) => { + let transaction = client.multi() + await Promise.all(data.map(async ({ kills, max_distance, total_distance, attacker_id }) => { + const prefix = genPrefix.player({ attacker_id }) + await processData(prefix, { kills, max_distance, total_distance }, transaction) + })) + return transaction.exec() + }) + //Global player deaths + await db.selectFrom('kill').select([count('id').as('kills'), 'victim_id']).groupBy('victim_id').execute() + .then(async (data) => { + let transaction = client.multi() + await Promise.all(data.map(async ({ kills, victim_id }) => { + const prefix = genPrefix.player({ attacker_id: victim_id }) + transaction.json.set('kills', prefix + ".deaths", Number(kills)) + })) + return transaction.exec() + }) + + //Player weapon kills + await db.selectFrom('kill').select([count('id').as('kills'), max('distance').as('max_distance'), sum('distance').as('total_distance'), 'attacker_id', 'cause_of_death']).groupBy(['attacker_id', 'cause_of_death']).whereRef("attacker_id", '!=', 'victim_id').execute() + .then(async (data) => { + let transaction = client.multi() + await Promise.all(data.map(async ({ kills, max_distance, total_distance, attacker_id, cause_of_death }) => { + await processData(genPrefix.playerWeapons({ attacker_id, cause_of_death }), { kills, max_distance, total_distance }, transaction) + await processData(genPrefix.weaponPlayers({ attacker_id, cause_of_death }), { kills, max_distance, total_distance }, transaction) + })) + return transaction.exec() + }) + //player weapon deaths + await db.selectFrom('kill').select([count('id').as('kills'), 'victim_id', 'cause_of_death']).groupBy(['victim_id', 'cause_of_death']).execute() + .then(async (data) => { + let transaction = client.multi() + await Promise.all(data.map(async ({ kills, victim_id, cause_of_death }) => { + transaction.json.set('kills', genPrefix.weaponPlayers({ attacker_id: victim_id, cause_of_death }) + ".deaths", Number(kills)) + })) + return transaction.exec() + }) +} + +async function processServerStats() { + let promises: Promise[] = [] + //Server global kills + await db.selectFrom('kill') + .select([count('id').as('kills'), max('distance').as('max_distance'), sum('distance').as('total_distance'), 'server']) + .groupBy(['server']).execute().then(async (data) => { + let transaction = client.multi() + const p = data.map(async ({ kills, max_distance, total_distance, server }) => { + if (!await client.json.type('kills', `servers.${server}`)) await client.json.set('kills', `servers.${server}`, { data: {}, weapons: {}, players: {} }) + await processData(genPrefix.global({ server }), { total_distance, max_distance, kills }, transaction) + }) + await Promise.all(p) + return transaction.exec() + }) + + //Server weapon kills + await db.selectFrom('kill') + .select([count('id').as('kills'), max('distance').as('max_distance'), sum('distance').as('total_distance'), 'cause_of_death', 'server']).groupBy(['cause_of_death', 'server']).execute().then(async (data) => { + let transaction = client.multi() + await Promise.all(data.map(async ({ kills, max_distance, total_distance, cause_of_death, server }) => { + const prefix = genPrefix.weapon({ cause_of_death, server }) + await processData(prefix, { kills, max_distance, total_distance }, transaction) + })) + return transaction.exec() + }) + //server player kill + await db.selectFrom('kill').select([count('id').as('kills'), max('distance').as('max_distance'), sum('distance').as('total_distance'), 'attacker_id', 'server']).groupBy(['attacker_id', 'server']).whereRef("attacker_id", '!=', 'victim_id').execute() + .then(async (data) => { + let transaction = client.multi() + await Promise.all(data.map(async ({ kills, max_distance, total_distance, attacker_id, server }) => { + const prefix = genPrefix.player({ attacker_id, server }) + await processData(prefix, { kills, max_distance, total_distance }, transaction) + })) + return transaction.exec() + }) + //server player deaths + await db.selectFrom('kill').select([count('id').as('kills'), 'victim_id', 'server']).groupBy(['victim_id', 'server']).execute() + .then(async (data) => { + let transaction = client.multi() + await Promise.all(data.map(async ({ kills, victim_id, server }) => { + const prefix = genPrefix.player({ attacker_id: victim_id, server }) + transaction.json.set('kills', prefix + ".deaths", Number(kills)) + })) + return transaction.exec() + }) + + //player weapon kills + await db.selectFrom('kill').select([count('id').as('kills'), max('distance').as('max_distance'), sum('distance').as('total_distance'), 'attacker_id', 'cause_of_death', 'server']).groupBy(['attacker_id', 'cause_of_death', 'server']).whereRef("attacker_id", '!=', 'victim_id').execute() + .then(async (data) => { + let transaction = client.multi() + await Promise.all(data.map(async ({ kills, max_distance, total_distance, attacker_id, cause_of_death, server }) => { + await processData(genPrefix.playerWeapons({ attacker_id, cause_of_death, server }), { kills, max_distance, total_distance }, transaction) + await processData(genPrefix.weaponPlayers({ attacker_id, cause_of_death, server }), { kills, max_distance, total_distance }, transaction) + })) + return transaction.exec() + }) + //player weapon deaths + await db.selectFrom('kill').select([count('id').as('kills'), 'victim_id', 'cause_of_death', 'server']).groupBy(['victim_id', 'cause_of_death', 'server']).execute() + .then(async (data) => { + let transaction = client.multi() + await Promise.all(data.map(async ({ kills, victim_id, cause_of_death, server }) => { + transaction.json.set('kills', genPrefix.weaponPlayers({ attacker_id: victim_id, cause_of_death, server }) + ".deaths", Number(kills)) + })) + return transaction.exec() + }) +} + +async function processData(prefix: string, { total_distance, max_distance, kills }: { total_distance: string | number | bigint, max_distance: number, kills: string | number | bigint }, transaction: ReturnType) { + if (!await client.json.type('kills', prefix)) await client.json.set('kills', prefix, {}) + transaction = transaction.json.set('kills', prefix + ".total_distance", Number(total_distance)) + transaction = transaction.json.set('kills', prefix + ".max_distance", Number(max_distance)) + transaction = transaction.json.set('kills', prefix + ".kills", Number(kills)) + return +} + +export default processAll diff --git a/src/processMain.ts b/src/processMain.ts index 76ae4d7..5be088d 100644 --- a/src/processMain.ts +++ b/src/processMain.ts @@ -2,11 +2,13 @@ import * as dotenv from 'dotenv' dotenv.config() import { dbReady } from './db/db' import { cacheReady } from './cache/redis' -import processAll from './client/process' +import processAll from './process/process' +import listenKills from "./process/onKill" async function startup() { await dbReady() await cacheReady() + await listenKills() await processAll() } export default startup() \ No newline at end of file diff --git a/tests/client.test.ts b/tests/client.test.ts index c118a60..c5a0adf 100644 --- a/tests/client.test.ts +++ b/tests/client.test.ts @@ -28,6 +28,24 @@ describe('client', () => { expect(first[1]).toHaveProperty('kills') }) + test('player list with weapon filter', async () => { + const request = await fetch("http://127.0.0.1:3000/players?weapons=sniper") + const data = await request.json() + const first = Object.entries(data)[0] + expect(first[1]).toHaveProperty('max_distance') + expect(first[1]).toHaveProperty('total_distance') + expect(first[1]).toHaveProperty('kills') + }) + + test('player list with weapon and server filter', async () => { + const request = await fetch("http://127.0.0.1:3000/players?weapons=sniper&server=1") + const data = await request.json() + const first = Object.entries(data)[0] + expect(first[1]).toHaveProperty('max_distance') + expect(first[1]).toHaveProperty('total_distance') + expect(first[1]).toHaveProperty('kills') + }) + test('weapon list', async () => { const request = await fetch("http://127.0.0.1:3000/weapons") const data = await request.json() @@ -36,6 +54,24 @@ describe('client', () => { expect(first[1]).toHaveProperty('total_distance') expect(first[1]).toHaveProperty('kills') }) + + test('weapon list with player filter', async () => { + const request = await fetch("http://127.0.0.1:3000/weapons?players=1005930844007") + const data = await request.json() + const first = Object.entries(data)[0] + expect(first[1]).toHaveProperty('max_distance') + expect(first[1]).toHaveProperty('total_distance') + expect(first[1]).toHaveProperty('kills') + }) + + test('weapon list with player and server filter', async () => { + const request = await fetch("http://127.0.0.1:3000/weapons?players=1005930844007&server=1") + const data = await request.json() + const first = Object.entries(data)[0] + expect(first[1]).toHaveProperty('max_distance') + expect(first[1]).toHaveProperty('total_distance') + expect(first[1]).toHaveProperty('kills') + }) }) afterAll((done) => { diff --git a/tests/realtime.test.ts b/tests/realtime.test.ts new file mode 100644 index 0000000..179278b --- /dev/null +++ b/tests/realtime.test.ts @@ -0,0 +1,110 @@ +import { afterAll, beforeAll, describe, expect, test } from '@jest/globals' +import clientMain from '../src/clientMain' +import serverMain from '../src/serverMain' +import listenKills from "../src/process/onKill" +import * as dotenv from 'dotenv' +dotenv.config() + +let listenServerClient + +const data = { + attacker_weapon_1_mods: 0, + victim_id: '0', + victim_name: 'TestVictim', + victim_offhand_weapon_2: 0, + attacker_offhand_weapon_3: '0', + victim_weapon_3_mods: 0, + attacker_weapon_2_mods: 0, + attacker_offhand_weapon_1: 0, + attacker_weapon_3_mods: 0, + attacker_offhand_weapon_2: 0, + victim_offhand_weapon_3: '0', + victim_offhand_weapon_1: 0, + attacker_weapon_3: 'defender', + killstat_version: 'ks_3.0.0', + attacker_weapon_1: 'smr', + match_id: '31b1f34d', + distance: 0, + victim_current_weapon: 'smr', + cause_of_death: 'smr', + victim_weapon_2_mods: 0, + victim_current_weapon_mods: 0, + attacker_current_weapon_mods: 0, + game_time: 377.799, + player_count: 1, + attacker_current_weapon: 'smr', + attacker_id: '1', + game_mode: 'tdm', + map: 'thaw', + attacker_weapon_2: 'autopistol', + victim_weapon_1: 'smr', + victim_weapon_2: 'autopistol', + victim_weapon_1_mods: 0, + victim_weapon_3: 'defender', + attacker_name: 'TestAttacker' +} + +beforeAll(async () => { + listenServerClient = await clientMain; + listenServerClient = await serverMain; + const response = await fetch(`http://127.0.0.1:3001/${process.env.SERVERAUTH_ID}/kill`, { + method: "POST", // *GET, POST, PUT, DELETE, etc. + credentials: "same-origin", // include, *same-origin, omit + headers: { + "Content-Type": "application/json", + 'Authorization': `Basic ${Buffer.from(process.env.SERVERAUTH_ID + ':' + process.env.SERVERAUTH_TOKEN).toString('base64')}` + }, + body: JSON.stringify(data), // body data type must match "Content-Type" header + }); + expect(response.status).toBe(201) +}) + +describe('realtime', () => { + let playerKills + let weaponKills + let playerDeaths + test('fetch player', async () => { + const request = await fetch("http://127.0.0.1:3000/players") + const data = await request.json() + const player = data["1"] + expect(player).toHaveProperty('max_distance') + expect(player).toHaveProperty('total_distance') + expect(player).toHaveProperty('kills') + playerKills = player.kills + playerDeaths = player.deaths + }) + + test('fetch weapon', async () => { + const request = await fetch("http://127.0.0.1:3000/weapons") + const data = await request.json() + const weapon = data.smr + expect(weapon).toHaveProperty('max_distance') + expect(weapon).toHaveProperty('total_distance') + expect(weapon).toHaveProperty('kills') + weaponKills = weapon.kills + }) + + test('update player', async () => { + const request = await fetch("http://127.0.0.1:3000/players") + const data = await request.json() + const player = data["1"] + expect(player).toHaveProperty('max_distance') + expect(player).toHaveProperty('total_distance') + expect(player).toHaveProperty('kills') + }) + + test('check player update', async () => { + const request = await fetch("http://127.0.0.1:3000/players") + const data = await request.json() + const player = data["1"] + expect(player.kills).toBe(playerKills + 1) + }) + + test('check weapon update', async () => { + const request = await fetch("http://127.0.0.1:3000/weapons") + const data = await request.json() + const weapon = data.smr + expect(weapon.kills).toBe(weaponKills + 1) + }) + +})