scraping working

.eslintrc.js

@@ -0,0 +1,14 @@
+module.exports = {
+  env: {
+    es2021: true,
+    node: true
+  },
+  extends: 'standard-with-typescript',
+  overrides: [],
+  parserOptions: {
+    ecmaVersion: 'latest',
+    sourceType: 'module',
+    project: ['./tsconfig.json'] // Specify it only for TypeScript files
+  },
+  rules: {}
+}

.gitignore

@@ -0,0 +1,132 @@
+# Logs
+logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+lerna-debug.log*
+.pnpm-debug.log*
+
+# Diagnostic reports (https://nodejs.org/api/report.html)
+report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json
+
+# Runtime data
+pids
+*.pid
+*.seed
+*.pid.lock
+
+# Directory for instrumented libs generated by jscoverage/JSCover
+lib-cov
+
+# Coverage directory used by tools like istanbul
+coverage
+*.lcov
+
+# nyc test coverage
+.nyc_output
+
+# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
+.grunt
+
+# Bower dependency directory (https://bower.io/)
+bower_components
+
+# node-waf configuration
+.lock-wscript
+
+# Compiled binary addons (https://nodejs.org/api/addons.html)
+build/Release
+
+# Dependency directories
+node_modules/
+jspm_packages/
+
+# Snowpack dependency directory (https://snowpack.dev/)
+web_modules/
+
+# TypeScript cache
+*.tsbuildinfo
+
+# Optional npm cache directory
+.npm
+
+# Optional eslint cache
+.eslintcache
+
+# Optional stylelint cache
+.stylelintcache
+
+# Microbundle cache
+.rpt2_cache/
+.rts2_cache_cjs/
+.rts2_cache_es/
+.rts2_cache_umd/
+
+# Optional REPL history
+.node_repl_history
+
+# Output of 'npm pack'
+*.tgz
+
+# Yarn Integrity file
+.yarn-integrity
+
+# dotenv environment variable files
+.env
+.env.development.local
+.env.test.local
+.env.production.local
+.env.local
+
+# parcel-bundler cache (https://parceljs.org/)
+.cache
+.parcel-cache
+
+# Next.js build output
+.next
+out
+
+# Nuxt.js build / generate output
+.nuxt
+dist
+
+# Gatsby files
+.cache/
+# Comment in the public line in if your project uses Gatsby and not Next.js
+# https://nextjs.org/blog/next-9-1#public-directory-support
+# public
+
+# vuepress build output
+.vuepress/dist
+
+# vuepress v2.x temp and cache directory
+.temp
+.cache
+
+# Docusaurus cache and generated files
+.docusaurus
+
+# Serverless directories
+.serverless/
+
+# FuseBox cache
+.fusebox/
+
+# DynamoDB Local files
+.dynamodb/
+
+# TernJS port file
+.tern-port
+
+# Stores VSCode versions used for testing VSCode extensions
+.vscode-test
+
+# yarn v2
+.yarn/cache
+.yarn/unplugged
+.yarn/build-state.yml
+.yarn/install-state.gz
+.pnp.*
+
+config.json

.vscode/launch.json

@@ -0,0 +1,26 @@
+{
+  // Use IntelliSense to learn about possible attributes.
+  // Hover to view descriptions of existing attributes.
+  // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+  "version": "0.2.0",
+  "configurations": [
+    {
+      "type": "node",
+      "request": "launch",
+      "name": "Launch Bot",
+      "skipFiles": ["<node_internals>/**"],
+      "program": "${workspaceFolder}/src/index.ts",
+      "preLaunchTask": "tsc: build - tsconfig.json",
+      "outFiles": ["${workspaceFolder}/out/**/*.js"]
+    },
+    {
+      "type": "node",
+      "request": "launch",
+      "name": "Sync Commands",
+      "skipFiles": ["<node_internals>/**"],
+      "program": "${workspaceFolder}/src/syncCommands.ts",
+      "preLaunchTask": "tsc: build - tsconfig.json",
+      "outFiles": ["${workspaceFolder}/out/**/*.js"]
+    }
+  ]
+}

.vscode/settings.json

@@ -0,0 +1,13 @@
+{
+  "editor.formatOnSave": true,
+  "typescript.preferences.quoteStyle": "single",
+  "javascript.preferences.quoteStyle": "single",
+  "editor.defaultFormatter": "dbaeumer.vscode-eslint",
+  "eslint.format.enable": true,
+  "[typescript]": {
+    "editor.defaultFormatter": "dbaeumer.vscode-eslint"
+  },
+  "[javascript]": {
+    "editor.defaultFormatter": "dbaeumer.vscode-eslint"
+  }
+}

package.json

@@ -0,0 +1,25 @@
+{
+  "name": "goobot",
+  "version": "1.0.0",
+  "description": "",
+  "main": "index.js",
+  "scripts": {
+    "build": "npx tsc",
+    "start": "node out/index.js"
+  },
+  "author": "",
+  "license": "ISC",
+  "devDependencies": {
+    "@typescript-eslint/eslint-plugin": "^5.59.11",
+    "eslint": "^8.42.0",
+    "eslint-config-standard-with-typescript": "^35.0.0",
+    "eslint-plugin-import": "^2.27.5",
+    "eslint-plugin-n": "^15.7.0",
+    "eslint-plugin-promise": "^6.1.1",
+    "typescript": "^5.1.3"
+  },
+  "dependencies": {
+    "discord.js": "^14.11.0",
+    "sqlite3": "^5.1.6"
+  }
+}

src/@types/discordjs.d.ts

@@ -0,0 +1,14 @@
+export interface SlashCommand {
+  data: SlashCommandBuilder
+  execute: (interaction: CommandInteraction) => Promise<void>
+}
+
+declare module 'discord.js' {
+  export interface Collection<K, V> extends Collection<K, V> {
+    filter: (value: V) => boolean
+  }
+
+  export interface Client<Ready extends boolean = boolean> extends Client<Ready> {
+    commands: Collection<K, SlashCommand>
+  }
+}

src/commands/load.ts

@@ -0,0 +1,56 @@
+import { type CommandInteraction, SlashCommandBuilder, type Message } from 'discord.js'
+import { owner_id } from '../config.json'
+import db from '../db'
+
+module.exports = {
+  data: new SlashCommandBuilder()
+    .setName('load')
+    .setDescription('Provides information about the user.'),
+  async execute (interaction: CommandInteraction) {
+    if (interaction.user.id !== owner_id) {
+      await interaction.reply({ content: 'You do not have access to this command!', ephemeral: true })
+    }
+    if (interaction.channel?.isDMBased() === true) {
+      await interaction.reply({ content: 'This command is disabled for DM channels', ephemeral: true })
+      return
+    }
+    let loaded = 0
+    const images = new Set<string>()
+    let messages = await interaction.channel?.messages.fetch().catch(console.error)
+    if (messages === undefined) {
+      await interaction.reply('Couldn\'t fetch messages')
+      return
+    }
+    loaded += messages.size
+    await interaction.reply(`Loading messages... loaded ${loaded} with ${images.size} images`)
+
+    while (messages !== undefined && messages.size > 0) {
+      const imagesURL = (messages as unknown as Message[]).map(e => e.attachments).filter(e => e.size > 0).map(e => e.map(v => v)).flat().filter(e => e.contentType?.startsWith('image')).map(e => e.url).flat()
+      imagesURL.forEach(images.add, images)
+      const lastmessage = messages.last() as Message | undefined
+      if (lastmessage === undefined) {
+        await interaction.editReply('Coudn\'t fetch message data')
+        break
+      }
+      await interaction.editReply(`Loading messages... loaded ${loaded} messages with ${images.size} images`)
+
+      messages = await interaction.channel?.messages.fetch({ before: lastmessage.id }).catch(console.error)
+      if (messages !== undefined) loaded += messages.size
+    }
+
+    await interaction.editReply(`Loading complete ! loaded ${loaded} messages with ${images.size} images`)
+    const stmt = db.prepare('INSERT INTO goob VALUES (?)')
+
+    images.forEach(e => stmt.run(e))
+    stmt.finalize()
+
+    db.each('SELECT rowid AS id, url FROM goob', (err, row: any) => {
+      console.log(`${row.id as string} : ${row.url as string}`)
+      if (err != null) console.error(err)
+    })
+    console.log(images)
+    // interaction.user is the object representing the User who ran the command
+    // interaction.member is the GuildMember object, which represents the user in the specific guild
+    // await interaction.reply(`This command was run by ${interaction.user.username}`)
+  }
+}

src/db/index.ts

@@ -0,0 +1,9 @@
+import { verbose } from 'sqlite3'
+const sqlite = verbose()
+const db = new sqlite.Database('goobers')
+
+db.serialize(() => {
+  db.run('CREATE TABLE IF NOT EXISTS goob (url TEXT UNIQUE)')
+})
+
+export default db

src/index.ts

@@ -0,0 +1,57 @@
+import { Client, Collection, Events, GatewayIntentBits } from 'discord.js'
+import { discord_token } from './config.json'
+import path from 'path'
+import fs from 'fs'
+
+// Create a new client instance
+const client = new Client({ intents: [GatewayIntentBits.Guilds] })
+client.commands = new Collection()
+
+// When the client is ready, run this code (only once)
+// We use 'c' for the event parameter to keep it separate from the already defined 'client'
+client.once(Events.ClientReady, c => {
+  console.log(`Ready! Logged in as ${c.user.tag}`)
+})
+
+const commandsPath = path.join(__dirname, 'commands')
+const commandFiles = fs.readdirSync(commandsPath).filter(file => file.endsWith('.js'))
+
+const promises = [] as Array<Promise<void>>
+
+for (const file of commandFiles) {
+  const filePath = path.join(commandsPath, file)
+  promises.push(import(filePath).then(({ default: command }) => {
+    if ('data' in command && 'execute' in command) {
+      client.commands.set(command.data.name, command)
+    } else {
+      console.log(`[WARNING] The command at ${filePath} is missing a required "data" or "execute" property.`)
+    }
+  }).catch(e => { console.log(e) }))
+  // Set a new item in the Collection with the key as the command name and the value as the exported module
+}
+
+client.on(Events.InteractionCreate, async interaction => {
+  if (!interaction.isChatInputCommand()) return
+
+  const command = interaction.client.commands.get(interaction.commandName)
+
+  if (command === undefined) {
+    console.error(`No command matching ${interaction.commandName} was found.`)
+    return
+  }
+
+  try {
+    await command.execute(interaction)
+  } catch (error) {
+    console.error(error)
+    if (interaction.replied || interaction.deferred) {
+      await interaction.followUp({ content: 'There was an error while executing this command!', ephemeral: true })
+    } else {
+      await interaction.reply({ content: 'There was an error while executing this command!', ephemeral: true })
+    }
+  }
+})
+
+// Log in to Discord with your client's token
+
+void Promise.all(promises).then(async () => await client.login(discord_token)).catch(e => { console.log(e) })

src/syncCommands.ts

@@ -0,0 +1,46 @@
+import { REST, Routes } from 'discord.js'
+import { discord_app_id, discord_guild_id, discord_token } from './config.json'
+import path from 'path'
+import fs from 'fs'
+import { type SlashCommand } from './@types/discordjs'
+
+const commands = [] as SlashCommand[]
+// Grab all the command files from the commands directory you created earlier
+const commandsPath = path.join(__dirname, 'commands')
+const commandFiles = fs.readdirSync(commandsPath).filter(file => file.endsWith('.js'))
+
+console.log(commandFiles)
+const promises = [] as Array<Promise<void>>
+for (const file of commandFiles) {
+  const filePath = path.join(commandsPath, file)
+  promises.push(import(filePath).then(({ default: command }) => {
+    if ('data' in command && 'execute' in command) {
+      commands.push(command)
+    } else {
+      console.log(`[WARNING] The command at ${filePath} is missing a required "data" or "execute" property.`)
+    }
+  }))
+  // Set a new item in the Collection with the key as the command name and the value as the exported module
+}
+
+// Construct and prepare an instance of the REST module
+const rest = new REST().setToken(discord_token)
+
+// and deploy your commands!
+
+void Promise.all(promises).then(async () => {
+  try {
+    console.log(`Started refreshing ${commands.length} application (/) commands.`)
+
+    // The put method is used to fully refresh all commands in the guild with the current set
+    const data = await rest.put(
+      Routes.applicationGuildCommands(discord_app_id, discord_guild_id),
+      { body: commands }
+    )
+
+    console.log(`Successfully reloaded ${(data as string[]).length} application (/) commands.`)
+  } catch (error) {
+    // And of course, make sure you catch and log any errors!
+    console.error(error)
+  }
+})

tsconfig.json

@@ -0,0 +1,108 @@
+{
+  "compilerOptions": {
+    /* Visit https://aka.ms/tsconfig to read more about this file */
+
+    /* Projects */
+    // "incremental": true,                              /* Save .tsbuildinfo files to allow for incremental compilation of projects. */
+    // "composite": true,                                /* Enable constraints that allow a TypeScript project to be used with project references. */
+    // "tsBuildInfoFile": "./.tsbuildinfo",              /* Specify the path to .tsbuildinfo incremental compilation file. */
+    // "disableSourceOfProjectReferenceRedirect": true,  /* Disable preferring source files instead of declaration files when referencing composite projects. */
+    // "disableSolutionSearching": true,                 /* Opt a project out of multi-project reference checking when editing. */
+    // "disableReferencedProjectLoad": true,             /* Reduce the number of projects loaded automatically by TypeScript. */
+
+    /* Language and Environment */
+    "target": "es2016" /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */,
+    // "lib": [],                                        /* Specify a set of bundled library declaration files that describe the target runtime environment. */
+    // "jsx": "preserve",                                /* Specify what JSX code is generated. */
+    // "experimentalDecorators": true,                   /* Enable experimental support for TC39 stage 2 draft decorators. */
+    // "emitDecoratorMetadata": true,                    /* Emit design-type metadata for decorated declarations in source files. */
+    // "jsxFactory": "",                                 /* Specify the JSX factory function used when targeting React JSX emit, e.g. 'React.createElement' or 'h'. */
+    // "jsxFragmentFactory": "",                         /* Specify the JSX Fragment reference used for fragments when targeting React JSX emit e.g. 'React.Fragment' or 'Fragment'. */
+    // "jsxImportSource": "",                            /* Specify module specifier used to import the JSX factory functions when using 'jsx: react-jsx*'. */
+    // "reactNamespace": "",                             /* Specify the object invoked for 'createElement'. This only applies when targeting 'react' JSX emit. */
+    // "noLib": true,                                    /* Disable including any library files, including the default lib.d.ts. */
+    // "useDefineForClassFields": true,                  /* Emit ECMAScript-standard-compliant class fields. */
+    // "moduleDetection": "auto",                        /* Control what method is used to detect module-format JS files. */
+
+    /* Modules */
+    "module": "commonjs" /* Specify what module code is generated. */,
+    // "rootDir": "./" /* Specify the root folder within your source files. */,
+    // "moduleResolution": "node",                       /* Specify how TypeScript looks up a file from a given module specifier. */
+    "baseUrl": "." /* Specify the base directory to resolve non-relative module names. */,
+    /* Specify a set of entries that re-map imports to additional lookup locations. */
+    // "rootDirs": [],                                   /* Allow multiple folders to be treated as one when resolving modules. */
+    "typeRoots": [
+      "node_modules/@types",
+      "src/@types"
+    ] /* Specify multiple folders that act like './node_modules/@types'. */,
+    // "types": [],                                      /* Specify type package names to be included without being referenced in a source file. */
+    // "allowUmdGlobalAccess": true,                     /* Allow accessing UMD globals from modules. */
+    // "moduleSuffixes": [],                             /* List of file name suffixes to search when resolving a module. */
+    "resolveJsonModule": true /* Enable importing .json files. */,
+    // "noResolve": true,                                /* Disallow 'import's, 'require's or '<reference>'s from expanding the number of files TypeScript should add to a project. */
+
+    /* JavaScript Support */
+    // "allowJs": true,                                  /* Allow JavaScript files to be a part of your program. Use the 'checkJS' option to get errors from these files. */
+    // "checkJs": true,                                  /* Enable error reporting in type-checked JavaScript files. */
+    // "maxNodeModuleJsDepth": 1,                        /* Specify the maximum folder depth used for checking JavaScript files from 'node_modules'. Only applicable with 'allowJs'. */
+
+    /* Emit */
+    // "declaration": true,                              /* Generate .d.ts files from TypeScript and JavaScript files in your project. */
+    // "declarationMap": true,                           /* Create sourcemaps for d.ts files. */
+    // "emitDeclarationOnly": true,                      /* Only output d.ts files and not JavaScript files. */
+    "sourceMap": true /* Create source map files for emitted JavaScript files. */,
+    // "outFile": "./",                                  /* Specify a file that bundles all outputs into one JavaScript file. If 'declaration' is true, also designates a file that bundles all .d.ts output. */
+    "outDir": "./out/" /* Specify an output folder for all emitted files. */,
+    // "removeComments": true,                           /* Disable emitting comments. */
+    // "noEmit": true,                                   /* Disable emitting files from a compilation. */
+    // "importHelpers": true,                            /* Allow importing helper functions from tslib once per project, instead of including them per-file. */
+    // "importsNotUsedAsValues": "remove",               /* Specify emit/checking behavior for imports that are only used for types. */
+    // "downlevelIteration": true,                       /* Emit more compliant, but verbose and less performant JavaScript for iteration. */
+    // "sourceRoot": "",                                 /* Specify the root path for debuggers to find the reference source code. */
+    // "mapRoot": "",                                    /* Specify the location where debugger should locate map files instead of generated locations. */
+    // "inlineSourceMap": true,                          /* Include sourcemap files inside the emitted JavaScript. */
+    // "inlineSources": true,                            /* Include source code in the sourcemaps inside the emitted JavaScript. */
+    // "emitBOM": true,                                  /* Emit a UTF-8 Byte Order Mark (BOM) in the beginning of output files. */
+    // "newLine": "crlf",                                /* Set the newline character for emitting files. */
+    // "stripInternal": true,                            /* Disable emitting declarations that have '@internal' in their JSDoc comments. */
+    // "noEmitHelpers": true,                            /* Disable generating custom helper functions like '__extends' in compiled output. */
+    // "noEmitOnError": true,                            /* Disable emitting files if any type checking errors are reported. */
+    // "preserveConstEnums": true,                       /* Disable erasing 'const enum' declarations in generated code. */
+    // "declarationDir": "./",                           /* Specify the output directory for generated declaration files. */
+    // "preserveValueImports": true,                     /* Preserve unused imported values in the JavaScript output that would otherwise be removed. */
+
+    /* Interop Constraints */
+    // "isolatedModules": true,                          /* Ensure that each file can be safely transpiled without relying on other imports. */
+    // "allowSyntheticDefaultImports": true,             /* Allow 'import x from y' when a module doesn't have a default export. */
+    "esModuleInterop": true /* Emit additional JavaScript to ease support for importing CommonJS modules. This enables 'allowSyntheticDefaultImports' for type compatibility. */,
+    // "preserveSymlinks": true,                         /* Disable resolving symlinks to their realpath. This correlates to the same flag in node. */
+    "forceConsistentCasingInFileNames": true /* Ensure that casing is correct in imports. */,
+
+    /* Type Checking */
+    "strict": true /* Enable all strict type-checking options. */,
+    // "noImplicitAny": true,                            /* Enable error reporting for expressions and declarations with an implied 'any' type. */
+    // "strictNullChecks": true,                         /* When type checking, take into account 'null' and 'undefined'. */
+    // "strictFunctionTypes": true,                      /* When assigning functions, check to ensure parameters and the return values are subtype-compatible. */
+    // "strictBindCallApply": true,                      /* Check that the arguments for 'bind', 'call', and 'apply' methods match the original function. */
+    // "strictPropertyInitialization": true,             /* Check for class properties that are declared but not set in the constructor. */
+    // "noImplicitThis": true,                           /* Enable error reporting when 'this' is given the type 'any'. */
+    // "useUnknownInCatchVariables": true,               /* Default catch clause variables as 'unknown' instead of 'any'. */
+    // "alwaysStrict": true,                             /* Ensure 'use strict' is always emitted. */
+    // "noUnusedLocals": true,                           /* Enable error reporting when local variables aren't read. */
+    // "noUnusedParameters": true,                       /* Raise an error when a function parameter isn't read. */
+    // "exactOptionalPropertyTypes": true,               /* Interpret optional property types as written, rather than adding 'undefined'. */
+    // "noImplicitReturns": true,                        /* Enable error reporting for codepaths that do not explicitly return in a function. */
+    // "noFallthroughCasesInSwitch": true,               /* Enable error reporting for fallthrough cases in switch statements. */
+    // "noUncheckedIndexedAccess": true,                 /* Add 'undefined' to a type when accessed using an index. */
+    // "noImplicitOverride": true,                       /* Ensure overriding members in derived classes are marked with an override modifier. */
+    // "noPropertyAccessFromIndexSignature": true,       /* Enforces using indexed accessors for keys declared using an indexed type. */
+    // "allowUnusedLabels": true,                        /* Disable error reporting for unused labels. */
+    // "allowUnreachableCode": true,                     /* Disable error reporting for unreachable code. */
+
+    /* Completeness */
+    // "skipDefaultLibCheck": true,                      /* Skip type checking .d.ts files that are included with TypeScript. */
+    "skipLibCheck": true /* Skip type checking all .d.ts files. */
+  },
+  "compileOnSave": true,
+  "exclude": ["jest.config.ts", "./tests"]
+}