smaller logs

build/public/index.html

@@ -4,7 +4,7 @@
   <meta charset="UTF-8">
   
   <title>🧩 jigsaw.hyottoko.club</title>
-<script type="module" crossorigin src="/assets/index.87048674.js"></script>
+<script type="module" crossorigin src="/assets/index.b2021c0c.js"></script>
   <link rel="modulepreload" href="/assets/vendor.684f7bc8.js">
   <link rel="stylesheet" href="/assets/index.8f0efd0f.css">
 </head>

build/server/main.js

@@ -1279,7 +1279,9 @@ const create = (gameId) => {
         const logfile = filename(gameId, 0);
         fs.appendFileSync(logfile, "");
         fs.appendFileSync(idxfile, JSON.stringify({
+            gameId: gameId,
             total: 0,
+            lastTs: 0,
             currentFile: logfile,
             perFile: LINES_PER_LOG_FILE,
         }));
@@ -1289,14 +1291,19 @@ const exists = (gameId) => {
     const idxfile = idxname(gameId);
     return fs.existsSync(idxfile);
 };
-const _log = (gameId, ...args) => {
+const _log = (gameId, type, ...args) => {
     const idxfile = idxname(gameId);
     if (!fs.existsSync(idxfile)) {
         return;
     }
+    const ts = args[args.length - 1];
+    const otherArgs = args.slice(0, -1);
     const idx = JSON.parse(fs.readFileSync(idxfile, 'utf-8'));
     idx.total++;
-    fs.appendFileSync(idx.currentFile, JSON.stringify(args) + "\n");
+    const diff = ts - idx.lastTs;
+    idx.lastTs = ts;
+    const line = JSON.stringify([type, ...otherArgs, diff]).slice(1, -1);
+    fs.appendFileSync(idx.currentFile, line + "\n");
     // prepare next log file
     if (idx.total % idx.perFile === 0) {
         const logfile = filename(gameId, idx.total);
@@ -1316,7 +1323,7 @@ const get = (gameId, offset = 0) => {
     }
     const log = fs.readFileSync(file, 'utf-8').split("\n");
     return log.filter(line => !!line).map(line => {
-        return JSON.parse(line);
+        return JSON.parse(`[${line}]`);
     });
 };
 var GameLog = {
@@ -1807,12 +1814,11 @@ async function createGame(gameId, targetTiles, image, ts, scoreMode, shapeMode,
 function addPlayer(gameId, playerId, ts) {
     if (GameLog.shouldLog(GameCommon.getFinishTs(gameId), ts)) {
         const idx = GameCommon.getPlayerIndexById(gameId, playerId);
-        const diff = ts - GameCommon.getStartTs(gameId);
         if (idx === -1) {
-            GameLog.log(gameId, Protocol.LOG_ADD_PLAYER, playerId, diff);
+            GameLog.log(gameId, Protocol.LOG_ADD_PLAYER, playerId, ts);
         }
         else {
-            GameLog.log(gameId, Protocol.LOG_UPDATE_PLAYER, idx, diff);
+            GameLog.log(gameId, Protocol.LOG_UPDATE_PLAYER, idx, ts);
         }
     }
     GameCommon.addPlayer(gameId, playerId, ts);
@@ -1821,8 +1827,7 @@ function addPlayer(gameId, playerId, ts) {
 function handleInput(gameId, playerId, input, ts) {
     if (GameLog.shouldLog(GameCommon.getFinishTs(gameId), ts)) {
         const idx = GameCommon.getPlayerIndexById(gameId, playerId);
-        const diff = ts - GameCommon.getStartTs(gameId);
+        GameLog.log(gameId, Protocol.LOG_HANDLE_INPUT, idx, input, ts);
-        GameLog.log(gameId, Protocol.LOG_HANDLE_INPUT, idx, input, diff);
     }
     const ret = GameCommon.handleInput(gameId, playerId, input, ts);
     GameStorage.setDirty(gameId);

scripts/rewrite_logs.ts

@@ -1,82 +0,0 @@
-import fs from 'fs'
-import Protocol from '../src/common/Protocol'
-import { logger } from '../src/common/Util'
-import { DATA_DIR } from '../src/server/Dirs'
-
-const log = logger('rewrite_logs')
-
-const filename = (gameId) => `${DATA_DIR}/log_${gameId}.log`
-
-const rewrite = (gameId) => {
-  const file = filename(gameId)
-  log.log(file)
-  if (!fs.existsSync(file)) {
-    return []
-  }
-  let playerIds = [];
-  let startTs = null
-  const lines = fs.readFileSync(file, 'utf-8').split("\n")
-  const linesNew = lines.filter(line => !!line).map((line) => {
-    const json = JSON.parse(line)
-    const m = {
-      createGame: Protocol.LOG_HEADER,
-      addPlayer: Protocol.LOG_ADD_PLAYER,
-      handleInput: Protocol.LOG_HANDLE_INPUT,
-    }
-    const action = json[0]
-    if (action in m) {
-      json[0] = m[action]
-      if (json[0] === Protocol.LOG_HANDLE_INPUT) {
-        const inputm = {
-          down: Protocol.INPUT_EV_MOUSE_DOWN,
-          up: Protocol.INPUT_EV_MOUSE_UP,
-          move: Protocol.INPUT_EV_MOUSE_MOVE,
-          zoomin: Protocol.INPUT_EV_ZOOM_IN,
-          zoomout: Protocol.INPUT_EV_ZOOM_OUT,
-          bg_color: Protocol.INPUT_EV_BG_COLOR,
-          player_color: Protocol.INPUT_EV_PLAYER_COLOR,
-          player_name: Protocol.INPUT_EV_PLAYER_NAME,
-        }
-        const inputa = json[2][0]
-        if (inputa in inputm) {
-          json[2][0] = inputm[inputa]
-        } else {
-          throw '[ invalid input log line: "' + line + '" ]'
-        }
-      }
-    } else {
-      throw '[ invalid general log line: "' + line + '" ]'
-    }
-
-    if (json[0] === Protocol.LOG_ADD_PLAYER) {
-      if (playerIds.indexOf(json[1]) === -1) {
-        playerIds.push(json[1])
-      } else {
-        json[0] = Protocol.LOG_UPDATE_PLAYER
-        json[1] = playerIds.indexOf(json[1])
-      }
-    }
-
-    if (json[0] === Protocol.LOG_HANDLE_INPUT) {
-      json[1] = playerIds.indexOf(json[1])
-      if (json[1] === -1) {
-        throw '[ invalid player ... "' + line + '" ]'
-      }
-    }
-
-    if (json[0] === Protocol.LOG_HEADER) {
-      startTs = json[json.length - 1]
-      json[4] = json[3]
-      json[3] = json[2]
-      json[2] = json[1]
-      json[1] = 1
-    } else {
-      json[json.length - 1] = json[json.length - 1] - startTs
-    }
-    return JSON.stringify(json)
-  })
-
-  fs.writeFileSync(file, linesNew.join("\n") + "\n")
-}
-
-rewrite(process.argv[2])

scripts/split_logs.ts

@@ -1,72 +1,73 @@
 import fs from 'fs'
-import readline from 'readline'
-import stream from 'stream'
 import { logger } from '../src/common/Util'
 import { DATA_DIR } from '../src/server/Dirs'
+import { filename } from '../src/server/GameLog'
 
 const log = logger('rewrite_logs')
 
-const doit = (file: string): Promise<void> => {
+interface IdxOld {
-  const filename = (offset: number) => file.replace(/\.log$/, `-${offset}.log`)
+  total: number
-  const idxname = () => file.replace(/\.log$/, `.idx.log`)
+  currentFile: string
-
+  perFile: number
-  let perfile = 10000
-  const idx = {
-    total: 0,
-    currentFile: '',
-    perFile: perfile,
-  }
-
-  return new Promise((resolve) => {
-    const instream = fs.createReadStream(DATA_DIR + '/' + file)
-    const outstream = new stream.Writable()
-    const rl = readline.createInterface(instream, outstream)
-
-
-    let lines: any[] = []
-    let offset = 0
-    let count = 0
-    rl.on('line', (line) => {
-      if (!line) {
-        // skip empty
-        return
-      }
-      count++
-      lines.push(line)
-      if (count >= perfile) {
-        const fn = filename(offset)
-        idx.currentFile = fn
-        idx.total += count
-        fs.writeFileSync(DATA_DIR + '/' + fn, lines.join("\n"))
-        count = 0
-        offset += perfile
-        lines = []
-      }
-    })
-
-    rl.on('close', () => {
-      if (count > 0) {
-        const fn = filename(offset)
-        idx.currentFile = fn
-        idx.total += count
-        fs.writeFileSync(DATA_DIR + '/' + fn, lines.join("\n"))
-        count = 0
-        offset += perfile
-        lines = []
-      }
-
-      fs.writeFileSync(DATA_DIR + '/' + idxname(), JSON.stringify(idx))
-      resolve()
-    })
-  })
 }
 
-let logs = fs.readdirSync(DATA_DIR)
+interface Idx {
-  .filter(f => f.toLowerCase().match(/^log_.*\.log$/))
+  gameId: string
+  total: number
+  lastTs: number
+  currentFile: string
+  perFile: number
+}
+const doit = (idxfile: string): void => {
+  const gameId: string = (idxfile.match(/^log_([a-z0-9]+)\.idx\.log$/) as any[])[1]
+  const idxOld: IdxOld = JSON.parse(fs.readFileSync(DATA_DIR + '/' + idxfile, 'utf-8'))
+
+  let currentFile = filename(gameId, 0)
+  const idxNew: Idx = {
+    gameId: gameId,
+    total: 0,
+    lastTs: 0,
+    currentFile: currentFile,
+    perFile: idxOld.perFile
+  }
+
+  let firstTs = 0
+  while (fs.existsSync(currentFile)) {
+    idxNew.currentFile = currentFile
+    const log = fs.readFileSync(currentFile, 'utf-8').split("\n")
+    const newLines = []
+    const lines = log.filter(line => !!line).map(line => {
+      return JSON.parse(line)
+    })
+    for (const l of lines) {
+      if (idxNew.total === 0) {
+        firstTs = l[4]
+        idxNew.lastTs = l[4]
+        newLines.push(JSON.stringify(l).slice(1, -1))
+      } else {
+        const ts = firstTs + l[l.length - 1]
+        const diff = ts - idxNew.lastTs
+        idxNew.lastTs = ts
+        const newL = l.slice(0, -1)
+        newL.push(diff)
+        newLines.push(JSON.stringify(newL).slice(1, -1))
+      }
+      idxNew.total++
+    }
+    fs.writeFileSync(idxNew.currentFile, newLines.join("\n") + "\n")
+    currentFile = filename(gameId, idxNew.total)
+  }
+
+  fs.writeFileSync(DATA_DIR + '/' + idxfile, JSON.stringify(idxNew))
+  console.log('done: ' + gameId)
+}
+
+let indexfiles = fs.readdirSync(DATA_DIR)
+  .filter(f => f.toLowerCase().match(/^log_[a-z0-9]+\.idx\.log$/))
 
 
 ;(async () => {
-  for (const file of logs) {
+  for (const file of indexfiles) {
     await doit(file)
   }
 })()

src/frontend/game.ts

@@ -590,6 +590,7 @@ export async function main(
       return false
     }
 
+    let GAME_TS = REPLAY.lastGameTs
     const next = async () => {
       if (REPLAY.logPointer + 1 >= REPLAY.log.length) {
         await queryNextReplayBatch(gameId)
@@ -614,18 +615,20 @@ export async function main(
         }
 
         const currLogEntry = REPLAY.log[REPLAY.logPointer]
-        const currTs: Timestamp = REPLAY.gameStartTs + currLogEntry[currLogEntry.length - 1]
+        const currTs: Timestamp = GAME_TS + currLogEntry[currLogEntry.length - 1]
+
         const nextLogEntry = REPLAY.log[nextIdx]
-        const nextTs: Timestamp = REPLAY.gameStartTs + nextLogEntry[nextLogEntry.length - 1]
+        const diffToNext = nextLogEntry[nextLogEntry.length - 1]
+        const nextTs: Timestamp = currTs + diffToNext
         if (nextTs > maxGameTs) {
           // next log entry is too far into the future
           if (REPLAY.skipNonActionPhases && (maxGameTs + 500 * Time.MS < nextTs)) {
-            const skipInterval = nextTs - currTs
+            maxGameTs += diffToNext
-            maxGameTs += skipInterval
           }
           break
         }
 
+        GAME_TS = currTs
         if (handleLogEntry(nextLogEntry, nextTs)) {
           RERENDER = true
         }

src/server/Game.ts

@@ -71,11 +71,10 @@ async function createGame(
 function addPlayer(gameId: string, playerId: string, ts: Timestamp): void {
   if (GameLog.shouldLog(GameCommon.getFinishTs(gameId), ts)) {
     const idx = GameCommon.getPlayerIndexById(gameId, playerId)
-    const diff = ts - GameCommon.getStartTs(gameId)
     if (idx === -1) {
-      GameLog.log(gameId, Protocol.LOG_ADD_PLAYER, playerId, diff)
+      GameLog.log(gameId, Protocol.LOG_ADD_PLAYER, playerId, ts)
     } else {
-      GameLog.log(gameId, Protocol.LOG_UPDATE_PLAYER, idx, diff)
+      GameLog.log(gameId, Protocol.LOG_UPDATE_PLAYER, idx, ts)
     }
   }
 
@@ -91,8 +90,7 @@ function handleInput(
 ): Array<Change> {
   if (GameLog.shouldLog(GameCommon.getFinishTs(gameId), ts)) {
     const idx = GameCommon.getPlayerIndexById(gameId, playerId)
-    const diff = ts - GameCommon.getStartTs(gameId)
+    GameLog.log(gameId, Protocol.LOG_HANDLE_INPUT, idx, input, ts)
-    GameLog.log(gameId, Protocol.LOG_HANDLE_INPUT, idx, input, diff)
   }
 
   const ret = GameCommon.handleInput(gameId, playerId, input, ts)

src/server/GameLog.ts

@@ -23,8 +23,8 @@ const shouldLog = (finishTs: Timestamp, currentTs: Timestamp): boolean => {
   return timeSinceGameEnd <= POST_GAME_LOG_DURATION
 }
 
-const filename = (gameId: string, offset: number) => `${DATA_DIR}/log_${gameId}-${offset}.log`
+export const filename = (gameId: string, offset: number) => `${DATA_DIR}/log_${gameId}-${offset}.log`
-const idxname = (gameId: string) => `${DATA_DIR}/log_${gameId}.idx.log`
+export const idxname = (gameId: string) => `${DATA_DIR}/log_${gameId}.idx.log`
 
 const create = (gameId: string): void => {
   const idxfile = idxname(gameId)
@@ -32,7 +32,9 @@ const create = (gameId: string): void => {
     const logfile = filename(gameId, 0)
     fs.appendFileSync(logfile, "")
     fs.appendFileSync(idxfile, JSON.stringify({
+      gameId: gameId,
       total: 0,
+      lastTs: 0,
       currentFile: logfile,
       perFile: LINES_PER_LOG_FILE,
     }))
@@ -44,15 +46,21 @@ const exists = (gameId: string): boolean => {
   return fs.existsSync(idxfile)
 }
 
-const _log = (gameId: string, ...args: Array<any>): void => {
+const _log = (gameId: string, type: number, ...args: Array<any>): void => {
   const idxfile = idxname(gameId)
   if (!fs.existsSync(idxfile)) {
     return
   }
 
+  const ts: Timestamp = args[args.length - 1]
+  const otherArgs: any[] = args.slice(0, -1)
+
   const idx = JSON.parse(fs.readFileSync(idxfile, 'utf-8'))
   idx.total++
-  fs.appendFileSync(idx.currentFile, JSON.stringify(args) + "\n")
+  const diff = ts - idx.lastTs
+  idx.lastTs = ts
+  const line = JSON.stringify([type, ...otherArgs, diff]).slice(1, -1)
+  fs.appendFileSync(idx.currentFile, line + "\n")
 
   // prepare next log file
   if (idx.total % idx.perFile === 0) {
@@ -79,7 +87,7 @@ const get = (
 
   const log = fs.readFileSync(file, 'utf-8').split("\n")
   return log.filter(line => !!line).map(line => {
-    return JSON.parse(line)
+    return JSON.parse(`[${line}]`)
   })
 }