diff --git a/build/server/main.js b/build/server/main.js index 098ff8d..618ce84 100644 --- a/build/server/main.js +++ b/build/server/main.js @@ -1730,6 +1730,9 @@ var GameSockets = { }; const log$1 = logger('Db.ts'); +// assume 32766 SQLITE_MAX_VARIABLE_NUMBER +// @see https://sqlite.org/limits.html +const SQLITE_MAX_VARIABLE_NUMBER = 32766; class Db { constructor(file, patchesDir) { this.file = file; @@ -1855,6 +1858,9 @@ class Db { } return this.get(table, check)[idcol]; // get id manually } + /** + * Inserts data into table and returns the last insert id + */ insert(table, data) { const keys = Object.keys(data); const values = keys.map(k => data[k]); @@ -1863,6 +1869,41 @@ class Db { + ' VALUES (' + keys.map(k => '?').join(',') + ')'; return this.run(sql, values).lastInsertRowid; } + /** + * Inserts multiple datas into table. Returns the total number + * of changes. + */ + insertMany(table, datas) { + if (datas.length === 0) { + return 0; + } + const keys = Object.keys(datas[0]); + const runChunk = (vars, values) => { + const sql = `INSERT INTO ${table} + (${keys.join(',')}) + VALUES ${vars.join(',')}`; + return this.run(sql, values).changes; + }; + let len = 0; + let vars = []; + let values = []; + let changes = 0; + for (const data of datas) { + if (len + keys.length > SQLITE_MAX_VARIABLE_NUMBER) { + changes += runChunk(vars, values); + len = 0; + vars = []; + values = []; + } + len += keys.length; + vars.push('(' + keys.map(_ => '?').join(',') + ')'); + values.push(...keys.map(k => data[k])); + } + if (len > 0) { + changes += runChunk(vars, values); + } + return changes; + } update(table, data, whereRaw = {}) { const keys = Object.keys(data); if (keys.length === 0) { diff --git a/scripts/import_game_logs.ts b/scripts/import_game_logs.ts new file mode 100644 index 0000000..8b88bb9 --- /dev/null +++ b/scripts/import_game_logs.ts @@ -0,0 +1,47 @@ +import { DB_FILE, DB_PATCHES_DIR, DATA_DIR } from '../src/server/Dirs' +import Db from '../src/server/Db' +import fs from 'fs' +import { logger } from '../src/common/Util' + +const log = logger('import_game_logs.ts') +const db = new Db(DB_FILE, DB_PATCHES_DIR) +db.patch(true) + + +for (const file of fs.readdirSync(DATA_DIR)) { + const m = file.match(/^log_(.*)\.log$/) + if (!m) { + continue + } + + const gameId = m[1] + log.info(`Importing log for game ${file}`) + + const contents = fs.readFileSync(`${DATA_DIR}/${file}`, 'utf-8') + let t = 0 + let datas = [] + for (const line of contents.split("\n")) { + if (line === '') { + continue + } + let parsed + try { + parsed = JSON.parse(line) + } catch (e) { + log.error('bad game', e) + break + } + if (t === 0) { + t = parsed[4] + } else { + t += parsed[parsed.length - 1] + } + datas.push({ + game_id: gameId, + created: t / 1000, + entry: line, + }) + } + db.insertMany('game_log', datas) + log.info(`Done.`) +} diff --git a/src/dbpatches/02_gamelog.sqlite b/src/dbpatches/02_gamelog.sqlite new file mode 100644 index 0000000..4fe0160 --- /dev/null +++ b/src/dbpatches/02_gamelog.sqlite @@ -0,0 +1,7 @@ +CREATE TABLE game_log ( + game_id TEXT, + + created TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + + entry TEXT NOT NULL +); diff --git a/src/server/Db.ts b/src/server/Db.ts index a3277f0..b61c9bd 100644 --- a/src/server/Db.ts +++ b/src/server/Db.ts @@ -5,6 +5,10 @@ import { logger } from '../common/Util' const log = logger('Db.ts') +// assume 32766 SQLITE_MAX_VARIABLE_NUMBER +// @see https://sqlite.org/limits.html +const SQLITE_MAX_VARIABLE_NUMBER = 32766 + /** * TODO: create a more specific type for OrderBy. * It looks like this (example): @@ -186,6 +190,9 @@ class Db { return this.get(table, check)[idcol] // get id manually } + /** + * Inserts data into table and returns the last insert id + */ insert (table: string, data: Data): Integer.IntLike { const keys = Object.keys(data) const values = keys.map(k => data[k]) @@ -195,6 +202,45 @@ class Db { return this.run(sql, values).lastInsertRowid } + /** + * Inserts multiple datas into table. Returns the total number + * of changes. + */ + insertMany (table: string, datas: Data[]): number { + if (datas.length === 0) { + return 0 + } + + const keys = Object.keys(datas[0]) + + const runChunk = (vars: string[], values: any[]) => { + const sql = `INSERT INTO ${table} + (${keys.join(',')}) + VALUES ${vars.join(',')}` + return this.run(sql, values).changes + } + + let len: number = 0 + let vars: string[] = [] + let values: any[] = [] + let changes = 0 + for (const data of datas) { + if (len + keys.length > SQLITE_MAX_VARIABLE_NUMBER) { + changes += runChunk(vars, values) + len = 0 + vars = [] + values = [] + } + len += keys.length + vars.push('(' + keys.map(_ => '?').join(',') + ')') + values.push(...keys.map(k => data[k])) + } + if (len > 0) { + changes += runChunk(vars, values) + } + return changes + } + update (table: string, data: Data, whereRaw: WhereRaw = {}): void { const keys = Object.keys(data) if (keys.length === 0) { diff --git a/src/server/main.ts b/src/server/main.ts index c5c89c4..7bc31cf 100644 --- a/src/server/main.ts +++ b/src/server/main.ts @@ -16,8 +16,7 @@ import { DB_FILE, DB_PATCHES_DIR, PUBLIC_DIR, - UPLOAD_DIR, - UPLOAD_URL + UPLOAD_DIR } from './Dirs' import { GameSettings, ScoreMode } from '../common/GameCommon' import GameStorage from './GameStorage'