add import script for existing game logs

build/server/main.js

@@ -1730,6 +1730,9 @@ var GameSockets = {
 };
 
 const log$1 = logger('Db.ts');
+// assume 32766 SQLITE_MAX_VARIABLE_NUMBER
+// @see https://sqlite.org/limits.html
+const SQLITE_MAX_VARIABLE_NUMBER = 32766;
 class Db {
     constructor(file, patchesDir) {
         this.file = file;
@@ -1855,6 +1858,9 @@ class Db {
         }
         return this.get(table, check)[idcol]; // get id manually
     }
+    /**
+     * Inserts data into table and returns the last insert id
+     */
     insert(table, data) {
         const keys = Object.keys(data);
         const values = keys.map(k => data[k]);
@@ -1863,6 +1869,41 @@ class Db {
             + ' VALUES (' + keys.map(k => '?').join(',') + ')';
         return this.run(sql, values).lastInsertRowid;
     }
+    /**
+     * Inserts multiple datas into table. Returns the total number
+     * of changes.
+     */
+    insertMany(table, datas) {
+        if (datas.length === 0) {
+            return 0;
+        }
+        const keys = Object.keys(datas[0]);
+        const runChunk = (vars, values) => {
+            const sql = `INSERT INTO ${table}
+        (${keys.join(',')})
+        VALUES ${vars.join(',')}`;
+            return this.run(sql, values).changes;
+        };
+        let len = 0;
+        let vars = [];
+        let values = [];
+        let changes = 0;
+        for (const data of datas) {
+            if (len + keys.length > SQLITE_MAX_VARIABLE_NUMBER) {
+                changes += runChunk(vars, values);
+                len = 0;
+                vars = [];
+                values = [];
+            }
+            len += keys.length;
+            vars.push('(' + keys.map(_ => '?').join(',') + ')');
+            values.push(...keys.map(k => data[k]));
+        }
+        if (len > 0) {
+            changes += runChunk(vars, values);
+        }
+        return changes;
+    }
     update(table, data, whereRaw = {}) {
         const keys = Object.keys(data);
         if (keys.length === 0) {

scripts/import_game_logs.ts

@@ -0,0 +1,47 @@
+import { DB_FILE, DB_PATCHES_DIR, DATA_DIR } from '../src/server/Dirs'
+import Db from '../src/server/Db'
+import fs from 'fs'
+import { logger } from '../src/common/Util'
+
+const log = logger('import_game_logs.ts')
+const db = new Db(DB_FILE, DB_PATCHES_DIR)
+db.patch(true)
+
+
+for (const file of fs.readdirSync(DATA_DIR)) {
+  const m = file.match(/^log_(.*)\.log$/)
+  if (!m) {
+    continue
+  }
+
+  const gameId = m[1]
+  log.info(`Importing log for game ${file}`)
+
+  const contents = fs.readFileSync(`${DATA_DIR}/${file}`, 'utf-8')
+  let t = 0
+  let datas = []
+  for (const line of contents.split("\n")) {
+    if (line === '') {
+      continue
+    }
+    let parsed
+    try {
+      parsed = JSON.parse(line)
+    } catch (e) {
+      log.error('bad game', e)
+      break
+    }
+    if (t === 0) {
+      t = parsed[4]
+    } else {
+      t += parsed[parsed.length - 1]
+    }
+    datas.push({
+      game_id: gameId,
+      created: t / 1000,
+      entry: line,
+    })
+  }
+  db.insertMany('game_log', datas)
+  log.info(`Done.`)
+}

src/dbpatches/02_gamelog.sqlite

@@ -0,0 +1,7 @@
+CREATE TABLE game_log (
+  game_id TEXT,
+
+  created TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+
+  entry TEXT NOT NULL
+);

src/server/Db.ts

@@ -5,6 +5,10 @@ import { logger } from '../common/Util'
 
 const log = logger('Db.ts')
 
+// assume 32766 SQLITE_MAX_VARIABLE_NUMBER
+// @see https://sqlite.org/limits.html
+const SQLITE_MAX_VARIABLE_NUMBER = 32766
+
 /**
  * TODO: create a more specific type for OrderBy.
  * It looks like this (example):
@@ -186,6 +190,9 @@ class Db {
     return this.get(table, check)[idcol] // get id manually
   }
 
+  /**
+   * Inserts data into table and returns the last insert id
+   */
   insert (table: string, data: Data): Integer.IntLike {
     const keys = Object.keys(data)
     const values = keys.map(k => data[k])
@@ -195,6 +202,45 @@ class Db {
     return this.run(sql, values).lastInsertRowid
   }
 
+  /**
+   * Inserts multiple datas into table. Returns the total number
+   * of changes.
+   */
+  insertMany (table: string, datas: Data[]): number {
+    if (datas.length === 0) {
+      return 0
+    }
+
+    const keys = Object.keys(datas[0])
+
+    const runChunk = (vars: string[], values: any[]) => {
+      const sql = `INSERT INTO ${table}
+        (${keys.join(',')})
+        VALUES ${vars.join(',')}`
+      return this.run(sql, values).changes
+    }
+
+    let len: number = 0
+    let vars: string[] = []
+    let values: any[] = []
+    let changes = 0
+    for (const data of datas) {
+      if (len + keys.length > SQLITE_MAX_VARIABLE_NUMBER) {
+        changes += runChunk(vars, values)
+        len = 0
+        vars = []
+        values = []
+      }
+      len += keys.length
+      vars.push('(' + keys.map(_ => '?').join(',') + ')')
+      values.push(...keys.map(k => data[k]))
+    }
+    if (len > 0) {
+      changes += runChunk(vars, values)
+    }
+    return changes
+  }
+
   update (table: string, data: Data, whereRaw: WhereRaw = {}): void {
     const keys = Object.keys(data)
     if (keys.length === 0) {

src/server/main.ts

@@ -16,8 +16,7 @@ import {
   DB_FILE,
   DB_PATCHES_DIR,
   PUBLIC_DIR,
-  UPLOAD_DIR,
-  UPLOAD_URL
+  UPLOAD_DIR
 } from './Dirs'
 import { GameSettings, ScoreMode } from '../common/GameCommon'
 import GameStorage from './GameStorage'