Merge pull request #16 from Legonzaur/v2

Add tests and optimize sql functions

.env.example

@@ -5,3 +5,4 @@ POSTGRES_PASSWORD="postgres"
 REDIS_USER="default"
 REDIS_PASSWORD=""
 REDIS_URL="redis://awesome.redis.server:6379"
+SERVERAUTH_TEST="1:f9f8fc8c-9185-47a8-8a99-9e3acf6ca007"

.vscode/launch.json

@@ -21,6 +21,15 @@
       "program": "${workspaceFolder}/src/clientMain.ts",
       "preLaunchTask": "tsc: build - tsconfig.json",
       "outFiles": ["${workspaceFolder}/out/**/*.js"]
+    },
+    {
+      "type": "node",
+      "request": "launch",
+      "name": "Launch Process API",
+      "skipFiles": ["<node_internals>/**"],
+      "program": "${workspaceFolder}/src/processMain.ts",
+      "preLaunchTask": "tsc: build - tsconfig.json",
+      "outFiles": ["${workspaceFolder}/out/**/*.js"]
     }
   ]
 }

babel.config.js

@@ -1,3 +1,6 @@
 module.exports = {
-  presets: [['@babel/preset-env', { targets: { node: 'current' } }]]
+  presets: [
+    ['@babel/preset-env', { targets: { node: 'current' } }],
+    '@babel/preset-typescript'
+  ]
 }

docs/v1.yml

@@ -271,7 +271,7 @@ paths:
       description: Used to test auth
       security:
         - Auth_Token: []
-  '/servers/{serverId}/kills':
+  '/servers/{serverId}/kill':
     post:
       summary: Post kills
       operationId: post-kills

package-lock.json

@@ -18,6 +18,7 @@
       "devDependencies": {
         "@babel/core": "^7.21.0",
         "@babel/preset-env": "^7.20.2",
+        "@babel/preset-typescript": "^7.21.4",
         "@jest/globals": "^29.4.3",
         "@types/cors": "^2.8.13",
         "@types/express": "^4.17.17",
@@ -1071,12 +1072,12 @@
       }
     },
     "node_modules/@babel/plugin-syntax-jsx": {
-      "version": "7.18.6",
-      "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-jsx/-/plugin-syntax-jsx-7.18.6.tgz",
-      "integrity": "sha512-6mmljtAedFGTWu2p/8WIORGwy+61PLgOMPOdazc7YoJ9ZCWUyFy3A6CpPkRKLKD1ToAesxX8KGEViAiLo9N+7Q==",
+      "version": "7.21.4",
+      "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-jsx/-/plugin-syntax-jsx-7.21.4.tgz",
+      "integrity": "sha512-5hewiLct5OKyh6PLKEYaFclcqtIgCb6bmELouxjF6up5q3Sov7rOayW4RwhbaBL0dit8rA80GNfY+UuDp2mBbQ==",
       "dev": true,
       "dependencies": {
-        "@babel/helper-plugin-utils": "^7.18.6"
+        "@babel/helper-plugin-utils": "^7.20.2"
       },
       "engines": {
         "node": ">=6.9.0"
@@ -1687,6 +1688,24 @@
         "@babel/core": "^7.0.0-0"
       }
     },
+    "node_modules/@babel/plugin-transform-typescript": {
+      "version": "7.21.3",
+      "resolved": "https://registry.npmjs.org/@babel/plugin-transform-typescript/-/plugin-transform-typescript-7.21.3.tgz",
+      "integrity": "sha512-RQxPz6Iqt8T0uw/WsJNReuBpWpBqs/n7mNo18sKLoTbMp+UrEekhH+pKSVC7gWz+DNjo9gryfV8YzCiT45RgMw==",
+      "dev": true,
+      "dependencies": {
+        "@babel/helper-annotate-as-pure": "^7.18.6",
+        "@babel/helper-create-class-features-plugin": "^7.21.0",
+        "@babel/helper-plugin-utils": "^7.20.2",
+        "@babel/plugin-syntax-typescript": "^7.20.0"
+      },
+      "engines": {
+        "node": ">=6.9.0"
+      },
+      "peerDependencies": {
+        "@babel/core": "^7.0.0-0"
+      }
+    },
     "node_modules/@babel/plugin-transform-unicode-escapes": {
       "version": "7.18.10",
       "resolved": "https://registry.npmjs.org/@babel/plugin-transform-unicode-escapes/-/plugin-transform-unicode-escapes-7.18.10.tgz",
@@ -1832,6 +1851,25 @@
         "@babel/core": "^7.0.0-0"
       }
     },
+    "node_modules/@babel/preset-typescript": {
+      "version": "7.21.4",
+      "resolved": "https://registry.npmjs.org/@babel/preset-typescript/-/preset-typescript-7.21.4.tgz",
+      "integrity": "sha512-sMLNWY37TCdRH/bJ6ZeeOH1nPuanED7Ai9Y/vH31IPqalioJ6ZNFUWONsakhv4r4n+I6gm5lmoE0olkgib/j/A==",
+      "dev": true,
+      "dependencies": {
+        "@babel/helper-plugin-utils": "^7.20.2",
+        "@babel/helper-validator-option": "^7.21.0",
+        "@babel/plugin-syntax-jsx": "^7.21.4",
+        "@babel/plugin-transform-modules-commonjs": "^7.21.2",
+        "@babel/plugin-transform-typescript": "^7.21.3"
+      },
+      "engines": {
+        "node": ">=6.9.0"
+      },
+      "peerDependencies": {
+        "@babel/core": "^7.0.0-0"
+      }
+    },
     "node_modules/@babel/regjsgen": {
       "version": "0.8.0",
       "resolved": "https://registry.npmjs.org/@babel/regjsgen/-/regjsgen-0.8.0.tgz",

package.json

@@ -13,6 +13,7 @@
   "devDependencies": {
     "@babel/core": "^7.21.0",
     "@babel/preset-env": "^7.20.2",
+    "@babel/preset-typescript": "^7.21.4",
     "@jest/globals": "^29.4.3",
     "@types/cors": "^2.8.13",
     "@types/express": "^4.17.17",

src/client/process.ts

@@ -1,15 +1,8 @@
 import db from '../db/db'
-import {
-  processWeaponReport,
-  populateWeaponSet,
-  processWeaponList
-} from './process/processWeapon'
-import {
-  populatePlayerSet,
-  processPlayerList,
-  processPlayerReport
-} from './process/processPlayer'
-import { getWeaponSet, getPlayerSet } from '../cache/cacheUtils'
+const { count, max, sum } = db.fn
+import client from '../cache/redis'
+import { createWeaponJson } from './process/processWeapon'
+import { createPlayerJson } from './process/processPlayer'
 
 /**
  * Starts the process of populating the REDIS database globally
@@ -18,33 +11,12 @@ import { getWeaponSet, getPlayerSet } from '../cache/cacheUtils'
 async function processAll() {
   console.log('Starting data calculation...')
   const timeStart = new Date()
-  // Iterate through all servers
-  const servers = await db.selectFrom('server').selectAll().execute()
-  await populateAllSets()
-  const promises: Promise<any>[] = []
-  servers.forEach((server) => {
-    promises.push(processServer(server.id))
-  })
-  //Iterate through all weapons
-  const weapons = await getWeaponSet()
-  weapons.forEach((weapon) => {
-    promises.push(processWeapon(weapon))
-  })
-  //Iterate through all players
-
-  const players = await getPlayerSet()
-  players.forEach((player) => {
-    promises.push(processPlayer(player))
-  })
-
-  await Promise.all(promises)
-  //Generate lists now that we have all data
-  await processAllLists()
 
+  await Promise.all([processGlobalStats(), processServerStats()])
   console.log(
     'Data calculation finished. Took + ' +
-      Math.abs(new Date().getTime() - timeStart.getTime()) / 1000 +
-      ' seconds'
+    Math.abs(new Date().getTime() - timeStart.getTime()) / 1000 +
+    ' seconds'
   )
   if (process.env.ENVIRONMENT == 'production') {
     return
@@ -52,157 +24,204 @@ async function processAll() {
   setTimeout(processAll, 3600000)
 }
 
-/**
- * Populte redis of server lists and reports
- * @param server
- */
-async function processServer(server: number) {
-  /*const promises: Promise<any>[] = []
-  //TODO TO REMOVE OR RENAME
-  await processServerPlayers(server)
-
-  //Iterate through all weapons present on this server
-  await populateWeaponSet(server)
-  const weapons = await cache.SMEMBERS(`servers:${server}:weapons`)
-  weapons.forEach((weapon) => {
-    promises.push(processWeaponReport(weapon, server))
-  })
-
-  //Iterate through all players present on this server (and do more stuff on those players)
-  const players = await cache.SMEMBERS(`servers:${server}:players`)
-  players.forEach((player) => {
-    promises.push(processPlayer({ server, player }))
-  })
-  await Promise.all(promises)
-  await processWeaponList(server)*/
-}
-
-/**
- * Populate redis of player lists and reports
- * @param param0
- */
-async function processPlayer(player: string) {
-  const promises: Promise<any>[] = []
-  //iterate through all servers
-  const servers = await db.selectFrom('server').selectAll().execute()
-  servers.forEach((server) => {
-    promises.push(
-      processPlayerReport(player, server.id),
-      //Iterate through all players on servers
-      getWeaponSet(server.id, player).then((weapons) => {
-        weapons.forEach((weapon) => {
-          promises.push(processPlayerReport(player, server.id, weapon))
-        })
+async function processGlobalStats() {
+  let promises: Promise<any>[] = []
+  //Global kills
+  promises.push(db.selectFrom('kill')
+    .select([count('id').as('kills'), max('distance').as('max_distance'), sum('distance').as('total_distance')])
+    .execute().then(data => {
+      let transaction = client.multi()
+      data.forEach(({ kills, max_distance, total_distance }) => {
+        transaction = processGlobalData({ kills, max_distance, total_distance }, transaction)
       })
-    )
-  })
-  //iterate through all weapons outside of servers
-  const weapons = await getWeaponSet(undefined, player)
-  weapons.forEach((weapon) => {
-    promises.push(processPlayerReport(player, undefined, weapon))
-  })
-  //global weapon data
-  promises.push(processPlayerReport(player))
-  await Promise.all(promises)
-}
+      return transaction.exec()
+    }))
 
-async function processWeapon(weapon: string) {
-  const promises: Promise<any>[] = []
-  //iterate through all servers
-  const servers = await db.selectFrom('server').selectAll().execute()
-  servers.forEach((server) => {
-    promises.push(
-      processWeaponReport(weapon, server.id),
-      //Iterate through all players on servers
-      getPlayerSet(server.id, weapon).then((players) => {
-        players.forEach((player) => {
-          promises.push(processWeaponReport(weapon, server.id, player))
-        })
+  //Global weapon kill
+  promises.push(db.selectFrom('kill')
+    .select([count('id').as('kills'), max('distance').as('max_distance'), sum('distance').as('total_distance'), 'cause_of_death'])
+    .groupBy('cause_of_death').execute().then(async (data) => {
+      let transaction = client.multi()
+      data.forEach(({ kills, max_distance, total_distance, cause_of_death }) => {
+        transaction = processWeaponData({ kills, max_distance, total_distance, cause_of_death }, transaction)
       })
-    )
-  })
-  //iterate through all players outside of servers
-  const players = await getPlayerSet(undefined, weapon)
-  players.forEach((player) => {
-    promises.push(processWeaponReport(weapon, undefined, player))
-  })
-  //global weapon data
-  promises.push(processWeaponReport(weapon))
-  await Promise.all(promises)
-}
+      await transaction.exec()
+    }))
 
-async function populateAllSets() {
-  const servers = await db.selectFrom('server').selectAll().execute()
-  await Promise.all([populatePlayerSet(), populateWeaponSet()])
-  await (async () => {
-    const promises: Promise<any>[] = []
-    await getWeaponSet().then((weaponSet) => {
-      weaponSet.forEach((weapon: string) => {
-        promises.push(populatePlayerSet(undefined, weapon))
+  //Player weapon kills
+  promises.push(db.selectFrom('kill')
+    .select([count('id').as('kills'), max('distance').as('max_distance'), sum('distance').as('total_distance'), 'cause_of_death', 'attacker_id'])
+    .groupBy(['cause_of_death', 'attacker_id']).execute().then(async (data) => {
+      let transaction = client.multi()
+      data.forEach(({ kills, max_distance, total_distance, cause_of_death, attacker_id }) => {
+        transaction = processWeaponData({ kills, max_distance, total_distance, cause_of_death, attacker_id }, transaction)
       })
-    })
-
-    await getPlayerSet().then((playerSet) => {
-      playerSet.forEach((player) => {
-        promises.push(populateWeaponSet(undefined, player))
+      await transaction.exec()
+    }))
+  //Player weapon deaths
+  promises.push(db.selectFrom('kill').select([count('id').as('deaths'), 'victim_id', 'cause_of_death'])
+    .groupBy(['victim_id', 'cause_of_death']).execute().then(data => {
+      let transaction = client.multi()
+      data.forEach(({ deaths, victim_id, cause_of_death }) => {
+        transaction = transaction.HSET(`weapons:${cause_of_death}:players:${victim_id}`, "deaths", deaths.toString())
       })
-    })
-    return Promise.all(promises)
-  })()
+      return transaction.exec()
+    }))
 
-  await (async () => {
-    const promises: Promise<any>[] = servers.map((server) =>
-      (async () => {
-        await Promise.all([
-          populatePlayerSet(server.id),
-          populateWeaponSet(server.id)
-        ])
-        const promises: Promise<any>[] = []
-        await getWeaponSet(server.id).then((weaponSet) => {
-          weaponSet.forEach((weapon: string) => {
-            promises.push(populatePlayerSet(server.id, weapon))
-          })
-        })
-        await getPlayerSet(server.id).then((playerSet) => {
-          playerSet.forEach((player) => {
-            promises.push(populateWeaponSet(server.id, player))
-          })
-        })
-        return Promise.all(promises)
-      })()
-    )
-    await Promise.all(promises)
-  })()
-}
-
-async function processAllLists() {
-  const promises: Promise<any>[] = []
-  promises.push(processPlayerList(), processWeaponList())
-  promises.push(
-    ...(await getPlayerSet()).map((player) =>
-      processWeaponList(undefined, player)
-    ),
-    ...(await (
-      await getWeaponSet()
-    ).map((weapon) => processPlayerList(undefined, weapon)))
-  )
-  const servers = await db.selectFrom('server').selectAll().execute()
-  promises.push(
-    ...servers.map(async ({ id }) => {
-      const promises: Promise<any>[] = []
-      promises.push(
-        processWeaponList(id),
-        processPlayerList(id),
-        ...(await getPlayerSet()).map((player) =>
-          processWeaponList(id, player)
-        ),
-        ...(await (
-          await getWeaponSet()
-        ).map((weapon) => processPlayerList(id, weapon)))
-      )
-      await Promise.all(promises)
+  //Global player kills
+  promises.push(db.selectFrom('kill')
+    .select([count('id').as('kills'), max('distance').as('max_distance'), sum('distance').as('total_distance'), 'attacker_id'])
+    .groupBy('attacker_id').execute().then(data => {
+      let transaction = client.multi()
+      data.forEach(({ kills, max_distance, total_distance, attacker_id }) => {
+        processPlayerData({ kills, max_distance, total_distance, attacker_id }, transaction)
+      })
+      return transaction.exec()
     })
   )
+  //Global player deaths
+  promises.push(db.selectFrom('kill').select([count('id').as('deaths'), 'victim_id'])
+    .groupBy('victim_id').execute().then(data => {
+      let transaction = client.multi()
+      data.forEach(({ deaths, victim_id }) => {
+        transaction = transaction.HSET('players:' + victim_id, "deaths", deaths.toString())
+      })
+      return transaction.exec()
+    }))
+
+  await Promise.all(promises)
+  promises = [];
+  promises.push(createWeaponJson());
+  promises.push(createPlayerJson());
+  (await client.sMembers('players')).forEach((player: string) => {
+    promises.push(createWeaponJson(undefined, player))
+  });
+  (await client.sMembers('weapons')).forEach((weapon: string) => {
+    promises.push(createPlayerJson(undefined, weapon))
+  });
   await Promise.all(promises)
 }
+
+async function processServerStats() {
+  let promises: Promise<any>[] = []
+  //Server global kills
+  promises.push(db.selectFrom('kill')
+    .select([count('id').as('kills'), max('distance').as('max_distance'), sum('distance').as('total_distance'), 'server'])
+    .groupBy(['server']).execute().then(data => {
+      let transaction = client.multi()
+      data.forEach(({ kills, max_distance, total_distance, server }) => {
+        transaction = processGlobalData({ kills, max_distance, total_distance, server }, transaction)
+      })
+      return transaction.exec()
+    }))
+  //Server weapon kills
+  promises.push(db.selectFrom('kill')
+    .select([count('id').as('kills'), max('distance').as('max_distance'), sum('distance').as('total_distance'), 'cause_of_death', 'server'])
+    .groupBy(['cause_of_death', 'server']).execute().then(data => {
+      let transaction = client.multi()
+      data.forEach(({ kills, max_distance, total_distance, cause_of_death, server }) => {
+        transaction = processWeaponData({ kills, max_distance, total_distance, cause_of_death, server }, transaction)
+      })
+      return transaction.exec()
+    }))
+
+  //Server Player weapon kills
+  promises.push(db.selectFrom('kill')
+    .select([count('id').as('kills'), max('distance').as('max_distance'), sum('distance').as('total_distance'), 'cause_of_death', 'attacker_id', 'server'])
+    .groupBy(['cause_of_death', 'attacker_id', 'server']).execute().then(data => {
+      let transaction = client.multi()
+      data.forEach(({ kills, max_distance, total_distance, cause_of_death, attacker_id, server }) => {
+        transaction = processWeaponData({ kills, max_distance, total_distance, cause_of_death, attacker_id, server }, transaction)
+      })
+      return transaction.exec()
+    }))
+  //Server Player weapon deaths
+  promises.push(db.selectFrom('kill').select([count('id').as('deaths'), 'victim_id', 'cause_of_death', 'server'])
+    .groupBy(['victim_id', 'cause_of_death', 'server']).execute().then(data => {
+      let transaction = client.multi()
+      data.forEach(({ deaths, victim_id, cause_of_death, server }) => {
+        transaction = transaction.HSET(`servers:${server}:weapons:${cause_of_death}:players:${victim_id}`, "deaths", deaths.toString())
+      })
+      return transaction.exec()
+    }))
+
+  //server player kills
+  promises.push(db.selectFrom('kill')
+    .select([count('id').as('kills'), max('distance').as('max_distance'), sum('distance').as('total_distance'), 'attacker_id', 'server'])
+    .groupBy(['attacker_id', 'server']).execute().then(data => {
+      let transaction = client.multi()
+      data.forEach(({ kills, max_distance, total_distance, attacker_id, server }) => {
+        transaction = processPlayerData({ kills, max_distance, total_distance, attacker_id, server }, transaction)
+      })
+      return transaction.exec()
+    })
+  )
+
+  //server player deaths
+  promises.push(db.selectFrom('kill').select([count('id').as('deaths'), 'victim_id', 'server'])
+    .groupBy(['victim_id', 'server']).execute().then(data => {
+      let transaction = client.multi()
+      data.forEach(({ deaths, victim_id, server }) => {
+        transaction = transaction.HSET('servers:' + server + ':players:' + victim_id, "deaths", deaths.toString())
+      })
+      return transaction.exec()
+    }))
+
+  await Promise.all(promises)
+  promises = [];
+  ((await client.sMembers('servers')).forEach((server: string) => {
+    if (isNaN(Number(server))) return
+    promises.push(createWeaponJson(Number(server)));
+    promises.push(createPlayerJson(Number(server)));
+    promises.push(client.sMembers('servers:' + server + ':players').then(data => {
+      data.forEach((player: string) => {
+        promises.push(createWeaponJson(Number(server), player))
+      })
+    }))
+    promises.push(client.sMembers('servers:' + server + 'weapons').then(data => {
+      data.forEach((weapon: string) => {
+        promises.push(createPlayerJson(Number(server), weapon))
+      })
+    }))
+  }));
+
+  await Promise.all(promises)
+}
+
+function processGlobalData({ kills, max_distance, total_distance, server }: { kills: string | number | bigint, max_distance: number, total_distance: string | number | bigint, server?: number }, transaction: any) {
+  const serverPrefix = server ? `servers:${server}:` : ''
+  const cacheLocation = serverPrefix + 'global'
+  transaction = transaction.HSET(cacheLocation, "total_distance", total_distance.toString())
+  transaction = transaction.HSET(cacheLocation, "max_distance", max_distance.toString())
+  transaction = transaction.HSET(cacheLocation, "kills", kills.toString())
+  if (server) {
+    transaction = transaction.SADD('servers', server.toString())
+  }
+
+  return transaction
+}
+
+function processWeaponData({ kills, max_distance, total_distance, cause_of_death, attacker_id, server }: { kills: string | number | bigint, max_distance: number, total_distance: string | number | bigint, cause_of_death: string, attacker_id?: string, server?: number }, transaction: any) {
+  const serverPrefix = server ? `servers:${server}:` : ''
+  const playerPrefix = attacker_id ? `players:${attacker_id}:` : ''
+  const cacheLocation = serverPrefix + playerPrefix + `weapons:` + cause_of_death
+  transaction = transaction.HSET(cacheLocation, "total_distance", total_distance.toString())
+  transaction = transaction.HSET(cacheLocation, "max_distance", max_distance.toString())
+  transaction = transaction.HSET(cacheLocation, "kills", kills.toString())
+  transaction = transaction.SADD(serverPrefix + playerPrefix + `weapons`, cause_of_death.toString())
+  return transaction
+}
+
+function processPlayerData({ kills, max_distance, total_distance, attacker_id, cause_of_death, server }: { kills: string | number | bigint, max_distance: number, total_distance: string | number | bigint, attacker_id: string, server?: number, cause_of_death?: string }, transaction: any) {
+  const serverPrefix = server ? `servers:${server}:` : ''
+  const weaponPrefix = cause_of_death ? `weapons:${cause_of_death}:` : ''
+  const cacheLocation = serverPrefix + weaponPrefix + `players:` + attacker_id
+
+  transaction = transaction.HSET(cacheLocation, "total_distance", total_distance.toString())
+  transaction = transaction.HSET(cacheLocation, "max_distance", max_distance.toString())
+  transaction = transaction.HSET(cacheLocation, "kills", kills.toString())
+  transaction = transaction.SADD(serverPrefix + weaponPrefix + `players`, attacker_id.toString())
+  return transaction
+}
+
 export default processAll

src/client/process/processPlayer.ts

@@ -1,208 +1,8 @@
 /**
  * This file contains all functions related to population of the REDIS database for players.
  */
-import db from '../../db/db'
 import cache from '../../cache/redis'
 import { getPlayerReport, getPlayerSet } from '../../cache/cacheUtils'
-import { sql } from 'kysely'
-const { count, max, avg } = db.fn
-
-/**
- * Populates the set of players (just a list with player IDS)
- *
- * Inputs `POSTGRESQL`
- *
- * Outputs `SET` `[servers:{serverId}:][weapons:{weaponId}:]players`
- * @param server optional server filter
- * @param weapon optional weapon filter
- * @returns
- */
-export async function populatePlayerSet(server?: number, weapon?: string) {
-  const serverPrefix = !isNaN(Number(server?.toString()))
-    ? `servers:${server}:`
-    : ''
-  const weaponPrefix = weapon ? `weapons:${weapon}:` : ''
-  const cacheLocation = serverPrefix + weaponPrefix + `players`
-  const last_entry = Number(await cache.GET(cacheLocation + ':last_entry')) || 0
-
-  //processKills
-  let query = db
-    .selectFrom('kill')
-    .select([
-      'attacker_id',
-      db
-        .selectFrom('kill')
-        .select(max('kill.id').as('last_entry'))
-        .as('last_entry')
-    ])
-    .groupBy('attacker_id')
-    .where('kill.id', '>', last_entry)
-  if (server) {
-    query = query.where('server', '=', server)
-  }
-  if (weapon) {
-    query = query.where('cause_of_death', '=', weapon)
-  }
-  let newData = await query.execute()
-  if (newData.length == 0) {
-    return
-  }
-  const promises: Promise<any>[] = []
-  newData.forEach(({ attacker_id }) => {
-    if (attacker_id == 'last_entry' || attacker_id == 'processedList') {
-      console.error('attacker_id cannot be ' + attacker_id)
-      return
-    }
-    promises.push(cache.SADD(cacheLocation, attacker_id))
-  })
-
-  //processDeaths
-  let query2 = db
-    .selectFrom('kill')
-    .select([
-      sql<string>`DISTINCT(victim_id)`.as('victim_id'),
-      db
-        .selectFrom('kill')
-        .select(max('kill.id').as('last_entry'))
-        .as('last_entry')
-    ])
-    .where('kill.id', '>', last_entry)
-  if (server) {
-    query2 = query2.where('server', '=', server)
-  }
-  if (weapon) {
-    query2 = query2.where('cause_of_death', '=', weapon)
-  }
-  const newData2 = await query2.execute()
-  newData2.forEach(({ victim_id }) => {
-    if (victim_id == 'last_entry' || victim_id == 'processedList') {
-      console.error('victim_id cannot be ' + victim_id)
-      return
-    }
-    promises.push(cache.SADD(cacheLocation, victim_id))
-  })
-  newData = await query.execute()
-  promises.push(
-    cache.SET(cacheLocation + ':last_entry', newData2[0].last_entry)
-  )
-  await Promise.all(promises)
-}
-
-/**
- * Processes player kills with optional server and/or weapon filter.
- * Does not processes deaths from weapons.
- *
- * Inputs `POSTGRESQL`
- *
- * Outputs `HASH` `[servers:{serverId}:][weapon:{weaponId}:]players:{playerId}`
- * @param player weapon to generate the report for
- * @param server optional server filter
- * @param weapon optional player filter
- */
-
-export async function processPlayerReport(
-  player: string,
-  server?: number,
-  weapon?: string
-) {
-  if (player == 'last_entry' || player == 'processedList') {
-    console.error('Player cannot be ' + player)
-    return
-  }
-  const serverPrefix = server ? `servers:${server}:` : ''
-  const weaponPrefix = weapon ? `weapons:${weapon}:` : ''
-  const cacheLocation = serverPrefix + weaponPrefix + `players:` + player
-  const last_entry = Number(await cache.HGET(cacheLocation, 'last_entry')) || 0
-  let query = db
-    .selectFrom('kill')
-    .select([
-      sql<string>`percentile_disc(0) WITHIN GROUP (ORDER BY attacker_name) FILTER (where attacker_id = ${player})`.as(
-        'username'
-      ),
-      count<number>('id')
-        .filterWhere('attacker_id', '=', player)
-        .filterWhereRef('attacker_id', '!=', 'victim_id')
-        .as('kills'),
-      count<number>('id').filterWhere('victim_id', '=', player).as('deaths'),
-      avg<number>('kill.distance')
-        .filterWhere('attacker_id', '=', player)
-        .as('avg_kill_distance'),
-      max('kill.distance')
-        .filterWhere('attacker_id', '=', player)
-        .as('max_kill_distance'),
-      db
-        .selectFrom('kill')
-        .select(max('kill.id').as('last_entry'))
-        .as('last_entry')
-    ])
-    .where((qb) =>
-      qb.where('attacker_id', '=', player).orWhere('victim_id', '=', player)
-    )
-    .where('kill.id', '>', last_entry)
-  if (server) {
-    query = query.where('server', '=', server)
-  }
-  if (weapon) {
-    query = query.where('cause_of_death', '=', weapon)
-  }
-  const newData = await query.execute()
-  if (newData.length == 0) {
-    return
-  }
-  const promises: Promise<any>[] = []
-  newData.forEach(
-    ({ kills, avg_kill_distance, max_kill_distance, deaths, username }) => {
-      promises.push(
-        (async () => {
-          if (!username) {
-            await cache.HSET(
-              cacheLocation,
-              'username',
-              (await cache.HGET(`players:` + player, 'username')) || ''
-            )
-          } else {
-            await cache.HSET(cacheLocation, 'username', username)
-          }
-        })()
-      )
-      promises.push(
-        processAvg(cacheLocation, 'avg_kill_distance', {
-          newkills: kills || 0,
-          newavg: avg_kill_distance || 0
-        }).then((e) => cache.HINCRBY(cacheLocation, 'kills', kills || 0)),
-        cache.HINCRBY(cacheLocation, 'deaths', deaths || 0),
-
-        processMax(cacheLocation, 'max_kill_distance', max_kill_distance)
-      )
-    }
-  )
-
-  promises.push(cache.HSET(cacheLocation, 'last_entry', newData[0].last_entry))
-  await Promise.all(promises)
-}
-
-async function processAvg(
-  cacheLocation: string,
-  key: 'avg_kill_distance',
-  { newkills, newavg }: { newkills: number; newavg: number }
-) {
-  const oldKills = Number(await cache.HGET(cacheLocation, 'kills')) || 0
-  const oldaverage = Number(await cache.HGET(cacheLocation, key)) || 0
-  const totalkills = Number(oldKills) + Number(newkills)
-  const newAverage =
-    (oldKills / totalkills) * oldaverage + (newkills / totalkills) * newavg || 0
-  await cache.HSET(cacheLocation, key, newAverage)
-}
-
-async function processMax(
-  cacheLocation: string,
-  key: 'max_kill_distance',
-  newmax: number
-) {
-  const oldMax = Number(await cache.HGET(cacheLocation, key)) || 0
-  if (oldMax < newmax || oldMax == 0)
-    await cache.HSET(cacheLocation, key, newmax || 0)
-}
 
 /**
  * Combines player reports in a readable json string.
@@ -216,7 +16,7 @@ async function processMax(
  * @param weapon optional player filter
  * @returns
  */
-export async function processPlayerList(server?: number, weapon?: string) {
+export async function createPlayerJson(server?: number, weapon?: string) {
   const serverPrefix = server ? `servers:${server}:` : ''
   const weaponPrefix = weapon ? `weapons:${weapon}:` : ''
   const cacheLocation = serverPrefix + weaponPrefix + `players`

src/client/process/processWeapon.ts

@@ -1,149 +1,8 @@
 /**
  * This file contains all functions related to population of the REDIS database for weapons.
  */
-import db from '../../db/db'
 import cache from '../../cache/redis'
 import { getWeaponReport } from '../../cache/cacheUtils'
-const { count, max, avg } = db.fn
-
-/**
- * Populates the set of weapons (just a list with weapon names)
- *
- * Inputs `POSTGRESQL`
- *
- * Outputs `SET` `[servers:{serverId}:][players:{playerId}:]weapons`
- * @param server optional server filter
- * @param player optional player filter
- * @returns
- */
-export async function populateWeaponSet(server?: number, player?: string) {
-  const serverPrefix = server ? `servers:${server}:` : ''
-  const playerPrefix = player ? `players:${player}:` : ''
-  const cacheLocation = serverPrefix + playerPrefix + `weapons`
-  const last_entry = Number(await cache.GET(cacheLocation + ':last_entry')) || 0
-  let query = db
-    .selectFrom('kill')
-    .select([
-      'cause_of_death',
-      db
-        .selectFrom('kill')
-        .select(max('kill.id').as('last_entry'))
-        .as('last_entry')
-    ])
-    .groupBy('cause_of_death')
-    .where('kill.id', '>', last_entry)
-  if (server) {
-    query = query.where('server', '=', server)
-  }
-  if (player) {
-    query = query
-      .where('attacker_id', '=', player)
-      .orWhere('victim_id', '=', player)
-  }
-  const newData = await query.execute()
-  if (newData.length == 0) {
-    return
-  }
-  const promises: Promise<any>[] = []
-  newData.forEach(({ cause_of_death }) => {
-    if (cause_of_death == 'last_entry' || cause_of_death == 'processedList') {
-      console.error('cause_of_death cannot be ' + cause_of_death)
-      return
-    }
-    promises.push(cache.SADD(cacheLocation, cause_of_death))
-  })
-  promises.push(cache.SET(cacheLocation + ':last_entry', newData[0].last_entry))
-  await Promise.all(promises)
-}
-
-/**
- * Processes weapon kills with optional server and/or player filter.
- * Does not processes deaths from weapons.
- *
- * Inputs `POSTGRESQL`
- *
- * Outputs `HASH` `[servers:{serverId}:][players:{playerId}:]weapons:{weaponId}`
- * @param weapon weapon to generate the report for
- * @param server optional server filter
- * @param player optional player filter
- */
-
-export async function processWeaponReport(
-  weapon: string,
-  server?: number,
-  player?: string
-) {
-  if (weapon == 'last_entry' || weapon == 'processedList') {
-    console.error('Weapon cannot be ' + weapon)
-    return
-  }
-  const serverPrefix = server ? `servers:${server}:` : ''
-  const playerPrefix = player ? `players:${player}:` : ''
-  const cacheLocation = serverPrefix + playerPrefix + `weapons:` + weapon
-  const last_entry = Number(await cache.HGET(cacheLocation, 'last_entry')) || 0
-  let query = db
-    .selectFrom('kill')
-    .select([
-      count<number>('id').as('kills'),
-      avg<number>('kill.distance').as('avg_kill_distance'),
-      max('kill.distance').as('max_kill_distance'),
-      db
-        .selectFrom('kill')
-        .select(max('kill.id').as('last_entry'))
-        .as('last_entry')
-    ])
-    .where('cause_of_death', '=', weapon)
-    .whereRef('attacker_id', '!=', 'victim_id')
-    .where('kill.id', '>', last_entry)
-    .groupBy('cause_of_death')
-  if (server) {
-    query = query.where('server', '=', server)
-  }
-  if (player) {
-    query = query.where('attacker_id', '=', player)
-  }
-  const newData = await query.execute()
-
-  if (newData.length == 0) {
-    return
-  }
-  const promises: Promise<any>[] = []
-  newData.forEach(({ kills, avg_kill_distance, max_kill_distance }) => {
-    promises.push(
-      processAvg(cacheLocation, 'avg_kill_distance', {
-        newkills: kills,
-        newavg: avg_kill_distance
-      }).then((e) => cache.HINCRBY(cacheLocation, 'kills', kills)),
-      processMax(cacheLocation, 'max_kill_distance', max_kill_distance)
-    )
-  })
-  promises.push(cache.HSET(cacheLocation, 'last_entry', newData[0].last_entry))
-  await Promise.all(promises)
-}
-
-async function processAvg(
-  cacheLocation: string,
-  key: 'avg_kill_distance',
-  { newkills, newavg }: { newkills: number; newavg: number }
-) {
-  const oldKills = Number(await cache.HGET(cacheLocation, 'kills')) || 0
-  const oldaverage = Number(await cache.HGET(cacheLocation, key)) | 0
-  const totalkills = oldKills + newkills
-  const newAverage =
-    (oldKills / totalkills) * oldaverage + (newkills / totalkills) * newavg
-  await cache.HSET(cacheLocation, key, newAverage)
-}
-
-async function processMax(
-  cacheLocation: string,
-  key: 'max_kill_distance',
-  newmax: number
-) {
-  const oldMax = Number(await cache.HGET(cacheLocation, key)) || -1
-  newmax = newmax || 0
-  if (!oldMax || !newmax) console.log(oldMax, newmax)
-  if (oldMax < newmax) await cache.HSET(cacheLocation, key, newmax)
-}
 
 /**
  * Combines weapons reports in a readable json string.
@@ -157,7 +16,7 @@ async function processMax(
  * @param player optional player filter
  * @returns
  */
-export async function processWeaponList(server?: number, player?: string) {
+export async function createWeaponJson(server?: number, player?: string) {
   const serverPrefix = server ? `servers:${server}:` : ''
   const playerPrefix = player ? `players:${player}:` : ''
   const cacheLocation = serverPrefix + playerPrefix + `weapons`

src/clientMain.ts

@@ -18,10 +18,14 @@ app.get('/', (req, res) => {
 
 app.use('/', client)
 
-dbReady().then((e) => {
-  cacheReady().then((e) => {
-    app.listen(port, '0.0.0.0', () => {
-      console.log(`Tone client api listening on port ${port}`)
+export default
+  new Promise((resolve, reject) => {
+    dbReady().then((e) => {
+      cacheReady().then((e) => {
+        const listenServer = app.listen(port, '0.0.0.0', () => {
+          console.log(`Tone client api listening on port ${port}`)
+          resolve(listenServer)
+        })
+      })
     })
   })
-})

src/db/db.ts

@@ -1,8 +1,9 @@
+import * as dotenv from 'dotenv'
+dotenv.config()
 import { InsertObject, Kysely, PostgresDialect } from 'kysely'
 import { Pool } from 'pg'
 import migrateToLatest from './migrations'
 import Database, { KillTable } from './model'
-
 const migration = migrateToLatest()
 const db = new Kysely<Database>({
   dialect: new PostgresDialect({

src/db/migrations.ts

@@ -9,7 +9,6 @@ import * as path from "path";
 import { Pool } from "pg";
 import Database from "./model";
 
-console.log(process.env.POSTGRES_HOST);
 async function migrateToLatest() {
   const db = new Kysely<Database>({
     dialect: new PostgresDialect({

src/processMain.ts

@@ -4,8 +4,9 @@ import { dbReady } from './db/db'
 import { cacheReady } from './cache/redis'
 import processAll from './client/process'
 
-dbReady().then((e) => {
-  cacheReady().then((e) => {
-    processAll()
-  })
-})
+async function startup() {
+  await dbReady()
+  await cacheReady()
+  await processAll()
+}
+export default startup()

src/serverMain.ts

@@ -2,7 +2,7 @@ import * as dotenv from 'dotenv'
 dotenv.config()
 import express from 'express'
 import cors from 'cors'
-import { dbReady } from './db/db'
+import db, { dbReady } from './db/db'
 import server from './server/server'
 
 const app = express()
@@ -17,8 +17,15 @@ app.get('/', (req, res) => {
 
 app.use('/', server)
 
-dbReady().then((e) => {
-  app.listen(port, '0.0.0.0', () => {
-    console.log(`Tone server api listening on port ${port}`)
+export default new Promise((resolve, reject) => {
+  dbReady().then((e) => {
+    const listenServer = app.listen(port, '0.0.0.0', () => {
+      console.log(`Tone server api listening on port ${port}`)
+      resolve(listenServer)
+    })
   })
 })
+
+app.on('close', () => {
+
+})

tests/client.test.ts

@@ -0,0 +1,45 @@
+import { afterAll, beforeAll, describe, expect, test } from '@jest/globals'
+import clientMain from '../src/clientMain'
+import * as dotenv from 'dotenv'
+dotenv.config()
+
+let listenServer
+
+beforeAll(async () => {
+    listenServer = await clientMain;
+})
+
+describe('client', () => {
+    test('server list', async () => {
+        const request = await fetch("http://127.0.0.1:3000/servers")
+        const data = await request.json()
+        expect(data.length).toBeGreaterThan(0)
+        expect(data[0]).toHaveProperty('name')
+        expect(data[0]).toHaveProperty('id')
+        expect(data[0]).toHaveProperty('description')
+    })
+
+    test('player list', async () => {
+        const request = await fetch("http://127.0.0.1:3000/players")
+        const data = await request.json()
+        const first = Object.entries(data)[0]
+        expect(first[1]).toHaveProperty('max_distance')
+        expect(first[1]).toHaveProperty('total_distance')
+        expect(first[1]).toHaveProperty('kills')
+    })
+
+    test('weapon list', async () => {
+        const request = await fetch("http://127.0.0.1:3000/weapons")
+        const data = await request.json()
+        const first = Object.entries(data)[0]
+        expect(first[1]).toHaveProperty('max_distance')
+        expect(first[1]).toHaveProperty('total_distance')
+        expect(first[1]).toHaveProperty('kills')
+    })
+})
+
+afterAll((done) => {
+    listenServer.close(() => {
+        done()
+    })
+})

tests/registration.test.ts

@@ -1,28 +0,0 @@
-import { describe, expect, test } from '@jest/globals'
-
-describe('registration', () => {
-  test('garbage body', () => {
-    expect(3).toBe(3)
-  })
-  test('incorrect data format', () => {
-    expect(3).toBe(3)
-  })
-  test('duplicate server', () => {
-    expect(3).toBe(3)
-  })
-  test('invisible server', () => {
-    expect(3).toBe(3)
-  })
-  test('wrong verification', () => {
-    expect(3).toBe(3)
-  })
-  test('masterserver unreachable', () => {
-    expect(3).toBe(3)
-  })
-  test('registration successful', () => {
-    expect(3).toBe(3)
-  })
-  test('server deletion', () => {
-    expect(3).toBe(3)
-  })
-})

tests/server.test.ts

@@ -0,0 +1,132 @@
+import { afterAll, beforeAll, describe, expect, test } from '@jest/globals'
+import serverMain from '../src/serverMain'
+import db from "../src/db/db"
+import * as dotenv from 'dotenv'
+dotenv.config()
+
+let listenServer
+
+beforeAll(async () => {
+  listenServer = await serverMain;
+})
+
+describe('server', () => {
+  test('server auth prefetch', async () => {
+    const response = await fetch(`http://127.0.0.1:3001/${process.env.SERVERAUTH_ID}`, {
+      method: "POST", // *GET, POST, PUT, DELETE, etc.
+      credentials: "same-origin", // include, *same-origin, omit
+      headers: {
+        "Content-Type": "application/json",
+        'Authorization': `Basic ${Buffer.from(process.env.SERVERAUTH_ID + ':' + process.env.SERVERAUTH_TOKEN).toString('base64')}`
+      }
+    });
+    expect(response.status).toBe(200)
+  })
+
+  test('register a kill', async () => {
+    const data = {
+      attacker_weapon_1_mods: 0,
+      victim_id: '0',
+      victim_name: 'TestVictim',
+      victim_offhand_weapon_2: 0,
+      attacker_offhand_weapon_3: '0',
+      victim_weapon_3_mods: 0,
+      attacker_weapon_2_mods: 0,
+      attacker_offhand_weapon_1: 0,
+      attacker_weapon_3_mods: 0,
+      attacker_offhand_weapon_2: 0,
+      victim_offhand_weapon_3: '0',
+      victim_offhand_weapon_1: 0,
+      attacker_weapon_3: 'defender',
+      killstat_version: 'ks_3.0.0',
+      attacker_weapon_1: 'smr',
+      match_id: '31b1f34d',
+      distance: 0,
+      victim_current_weapon: 'smr',
+      cause_of_death: 'smr',
+      victim_weapon_2_mods: 0,
+      victim_current_weapon_mods: 0,
+      attacker_current_weapon_mods: 0,
+      game_time: 377.799,
+      player_count: 1,
+      attacker_current_weapon: 'smr',
+      attacker_id: '1',
+      game_mode: 'tdm',
+      map: 'thaw',
+      attacker_weapon_2: 'autopistol',
+      victim_weapon_1: 'smr',
+      victim_weapon_2: 'autopistol',
+      victim_weapon_1_mods: 0,
+      victim_weapon_3: 'defender',
+      attacker_name: 'TestAttacker'
+    }
+    const response = await fetch(`http://127.0.0.1:3001/${process.env.SERVERAUTH_ID}/kill`, {
+      method: "POST", // *GET, POST, PUT, DELETE, etc.
+      credentials: "same-origin", // include, *same-origin, omit
+      headers: {
+        "Content-Type": "application/json",
+        'Authorization': `Basic ${Buffer.from(process.env.SERVERAUTH_ID + ':' + process.env.SERVERAUTH_TOKEN).toString('base64')}`
+      },
+      body: JSON.stringify(data), // body data type must match "Content-Type" header
+    });
+    expect(response.status).toBe(201)
+  })
+
+
+  test('register a kill with missing data', async () => {
+    const data = {
+      attacker_weapon_1_mods: NaN,
+      victim_id: '0',
+      victim_name: 'TestVictim',
+      victim_offhand_weapon_2: NaN,
+      attacker_offhand_weapon_3: 'null',
+      victim_weapon_3_mods: 19,
+      attacker_weapon_2_mods: NaN,
+      attacker_offhand_weapon_1: NaN,
+      attacker_weapon_3_mods: NaN,
+      attacker_offhand_weapon_2: NaN,
+      victim_offhand_weapon_3: '0',
+      victim_offhand_weapon_1: 1,
+      attacker_weapon_3: 'null',
+      killstat_version: 'ks_3.0.0',
+      attacker_weapon_1: 'null',
+      match_id: '554fa7a3',
+      distance: 202,
+      victim_current_weapon: 'car',
+      cause_of_death: 'satchel',
+      victim_weapon_2_mods: NaN,
+      victim_current_weapon_mods: NaN,
+      attacker_current_weapon_mods: NaN,
+      game_time: 516.417,
+      player_count: 13,
+      attacker_current_weapon: 'null',
+      attacker_id: '1',
+      game_mode: 'ps',
+      map: 'grave',
+      attacker_weapon_2: 'null',
+      victim_weapon_1: 'car',
+      victim_weapon_2: 'autopistol',
+      victim_weapon_1_mods: NaN,
+      victim_weapon_3: 'arc_launcher',
+      attacker_name: 'TestAttacker'
+    }
+    const response = await fetch(`http://127.0.0.1:3001/${process.env.SERVERAUTH_ID}/kill`, {
+      method: "POST", // *GET, POST, PUT, DELETE, etc.
+      credentials: "same-origin", // include, *same-origin, omit
+      headers: {
+        "Content-Type": "application/json",
+        'Authorization': `Basic ${Buffer.from(process.env.SERVERAUTH_ID + ':' + process.env.SERVERAUTH_TOKEN).toString('base64')}`
+      },
+      body: JSON.stringify(data), // body data type must match "Content-Type" header
+    });
+    expect(response.status).toBe(201)
+  })
+})
+
+afterAll((done) => {
+  listenServer.close(() => {
+    db.deleteFrom('kill').where('attacker_id', '=', '0').orWhere('attacker_id', '=', '1').execute().then(() => {
+      db.destroy().then(() => { done() })
+    })
+  })
+})

tests/tess.test.ts

@@ -1,44 +0,0 @@
-import { describe, expect, test } from '@jest/globals'
-
-describe('server', () => {
-  test('register a kill', () => {
-    const data = {
-      attacker_weapon_1_mods: 0,
-      victim_id: '1005930844007',
-      victim_name: 'Legonzaur',
-      victim_offhand_weapon_2: 0,
-      attacker_offhand_weapon_3: '0',
-      victim_weapon_3_mods: 0,
-      attacker_weapon_2_mods: 0,
-      attacker_offhand_weapon_1: 0,
-      attacker_weapon_3_mods: 0,
-      attacker_offhand_weapon_2: 0,
-      victim_offhand_weapon_3: '0',
-      victim_offhand_weapon_1: 0,
-      attacker_weapon_3: 'defender',
-      killstat_version: 'ks_3.0.0',
-      attacker_weapon_1: 'smr',
-      match_id: '31b1f34d',
-      distance: 0,
-      victim_current_weapon: 'smr',
-      cause_of_death: 'smr',
-      victim_weapon_2_mods: 0,
-      victim_current_weapon_mods: 0,
-      attacker_current_weapon_mods: 0,
-      game_time: 377.799,
-      player_count: 1,
-      attacker_current_weapon: 'smr',
-      attacker_id: '1005930844007',
-      game_mode: 'tdm',
-      map: 'thaw',
-      attacker_weapon_2: 'autopistol',
-      victim_weapon_1: 'smr',
-      victim_weapon_2: 'autopistol',
-      victim_weapon_1_mods: 0,
-      victim_weapon_3: 'defender',
-      attacker_name: 'Legonzaur'
-    }
-
-    expect(3).toBe(3)
-  })
-})