split logs so that replay works for long games

build/public/index.html

@@ -4,7 +4,7 @@
   <meta charset="UTF-8">
   
   <title>🧩 jigsaw.hyottoko.club</title>
-<script type="module" crossorigin src="/assets/index.7efa4c6c.js"></script>
+<script type="module" crossorigin src="/assets/index.ab1d6e0f.js"></script>
   <link rel="modulepreload" href="/assets/vendor.684f7bc8.js">
   <link rel="stylesheet" href="/assets/index.8f0efd0f.css">
 </head>

build/server/main.js

@@ -3,8 +3,6 @@ import express from 'express';
 import compression from 'compression';
 import multer from 'multer';
 import fs from 'fs';
-import readline from 'readline';
-import stream from 'stream';
 import { fileURLToPath } from 'url';
 import { dirname } from 'path';
 import sizeOf from 'image-size';
@@ -1249,6 +1247,7 @@ const PUBLIC_DIR = `${BASE_DIR}/build/public/`;
 const DB_PATCHES_DIR = `${BASE_DIR}/src/dbpatches`;
 const DB_FILE = `${BASE_DIR}/data/db.sqlite`;
 
+const LINES_PER_LOG_FILE = 10000;
 const POST_GAME_LOG_DURATION = 5 * Time.MIN;
 const shouldLog = (finishTs, currentTs) => {
     // when not finished yet, always log
@@ -1260,54 +1259,52 @@ const shouldLog = (finishTs, currentTs) => {
     const timeSinceGameEnd = currentTs - finishTs;
     return timeSinceGameEnd <= POST_GAME_LOG_DURATION;
 };
-const filename = (gameId) => `${DATA_DIR}/log_${gameId}.log`;
+const filename = (gameId, offset) => `${DATA_DIR}/log_${gameId}-${offset}.log`;
+const idxname = (gameId) => `${DATA_DIR}/log_${gameId}.idx.log`;
 const create = (gameId) => {
-    const file = filename(gameId);
+    const idxfile = idxname(gameId);
-    if (!fs.existsSync(file)) {
+    if (!fs.existsSync(idxfile)) {
-        fs.appendFileSync(file, '');
+        const logfile = filename(gameId, 0);
+        fs.appendFileSync(logfile, "");
+        fs.appendFileSync(idxfile, JSON.stringify({
+            total: 0,
+            currentFile: logfile,
+            perFile: LINES_PER_LOG_FILE,
+        }));
     }
 };
 const exists = (gameId) => {
-    const file = filename(gameId);
+    const idxfile = idxname(gameId);
-    return fs.existsSync(file);
+    return fs.existsSync(idxfile);
 };
 const _log = (gameId, ...args) => {
-    const file = filename(gameId);
+    const idxfile = idxname(gameId);
-    if (!fs.existsSync(file)) {
+    if (!fs.existsSync(idxfile)) {
         return;
     }
-    const str = JSON.stringify(args);
+    const idx = JSON.parse(fs.readFileSync(idxfile, 'utf-8'));
-    fs.appendFileSync(file, str + "\n");
+    idx.total++;
+    fs.appendFileSync(idx.currentFile, JSON.stringify(args) + "\n");
+    // prepare next log file
+    if (idx.total % idx.perFile === 0) {
+        const logfile = filename(gameId, idx.total);
+        fs.appendFileSync(logfile, "");
+        idx.currentFile = logfile;
+    }
+    fs.writeFileSync(idxfile, JSON.stringify(idx));
 };
-const get = async (gameId, offset = 0, size = 10000) => {
+const get = (gameId, offset = 0) => {
-    const file = filename(gameId);
+    const idxfile = idxname(gameId);
+    if (!fs.existsSync(idxfile)) {
+        return [];
+    }
+    const file = filename(gameId, offset);
     if (!fs.existsSync(file)) {
         return [];
     }
-    return new Promise((resolve) => {
+    const log = fs.readFileSync(file, 'utf-8').split("\n");
-        const instream = fs.createReadStream(file);
+    return log.map(line => {
-        const outstream = new stream.Writable();
+        return JSON.parse(line);
-        const rl = readline.createInterface(instream, outstream);
-        const lines = [];
-        let i = -1;
-        rl.on('line', (line) => {
-            if (!line) {
-                // skip empty
-                return;
-            }
-            i++;
-            if (offset > i) {
-                return;
-            }
-            if (offset + size <= i) {
-                rl.close();
-                return;
-            }
-            lines.push(JSON.parse(line));
-        });
-        rl.on('close', () => {
-            resolve(lines);
-        });
     });
 };
 var GameLog = {
@@ -2061,7 +2058,7 @@ app.get('/api/replay-data', async (req, res) => {
         res.status(404).send({ reason: 'no log found' });
         return;
     }
-    const log = await GameLog.get(gameId, offset, size);
+    const log = GameLog.get(gameId, offset);
     let game = null;
     if (offset === 0) {
         // also need the game

scripts/split_logs.ts

@@ -0,0 +1,72 @@
+import fs from 'fs'
+import readline from 'readline'
+import stream from 'stream'
+import { logger } from '../src/common/Util'
+import { DATA_DIR } from '../src/server/Dirs'
+
+const log = logger('rewrite_logs')
+
+const doit = (file: string): Promise<void> => {
+  const filename = (offset: number) => file.replace(/\.log$/, `-${offset}.log`)
+  const idxname = () => file.replace(/\.log$/, `.idx.log`)
+
+  let perfile = 10000
+  const idx = {
+    total: 0,
+    currentFile: '',
+    perFile: perfile,
+  }
+
+  return new Promise((resolve) => {
+    const instream = fs.createReadStream(DATA_DIR + '/' + file)
+    const outstream = new stream.Writable()
+    const rl = readline.createInterface(instream, outstream)
+
+
+    let lines: any[] = []
+    let offset = 0
+    let count = 0
+    rl.on('line', (line) => {
+      if (!line) {
+        // skip empty
+        return
+      }
+      count++
+      lines.push(line)
+      if (count >= perfile) {
+        const fn = filename(offset)
+        idx.currentFile = fn
+        idx.total += count
+        fs.writeFileSync(DATA_DIR + '/' + fn, lines.join("\n"))
+        count = 0
+        offset += perfile
+        lines = []
+      }
+    })
+
+    rl.on('close', () => {
+      if (count > 0) {
+        const fn = filename(offset)
+        idx.currentFile = fn
+        idx.total += count
+        fs.writeFileSync(DATA_DIR + '/' + fn, lines.join("\n"))
+        count = 0
+        offset += perfile
+        lines = []
+      }
+
+      fs.writeFileSync(DATA_DIR + '/' + idxname(), JSON.stringify(idx))
+      resolve()
+    })
+  })
+}
+
+let logs = fs.readdirSync(DATA_DIR)
+  .filter(f => f.toLowerCase().match(/^log_.*\.log$/))
+
+
+;(async () => {
+  for (const file of logs) {
+    await doit(file)
+  }
+})()

src/frontend/Communication.ts

@@ -114,10 +114,9 @@ function connect(
 
 async function requestReplayData(
   gameId: string,
-  offset: number,
+  offset: number
-  size: number
 ): Promise<ReplayData> {
-  const args = { gameId, offset, size }
+  const args = { gameId, offset }
   const res = await fetch(`/api/replay-data${Util.asQueryArgs(args)}`)
   const json: ReplayData = await res.json()
   return json

src/frontend/game.ts

@@ -69,7 +69,6 @@ interface Replay {
   skipNonActionPhases: boolean
   //
   dataOffset: number
-  dataSize: number
 }
 
 const shouldDrawPiece = (piece: Piece) => {
@@ -301,9 +300,8 @@ export async function main(
     lastRealTs: 0,
     lastGameTs: 0,
     gameStartTs: 0,
-    skipNonActionPhases: false,
+    skipNonActionPhases: true,
     dataOffset: 0,
-    dataSize: 10000,
   }
 
   Communication.onConnectionStateChange((state) => {
@@ -314,11 +312,10 @@ export async function main(
     gameId: string
   ): Promise<ReplayData> => {
     const offset = REPLAY.dataOffset
-    REPLAY.dataOffset += REPLAY.dataSize
+    REPLAY.dataOffset += 10000 // meh
     const replay: ReplayData = await Communication.requestReplayData(
       gameId,
-      offset,
+      offset
-      REPLAY.dataSize
     )
 
     // cut log that was already handled
@@ -326,7 +323,7 @@ export async function main(
     REPLAY.logPointer = 0
     REPLAY.log.push(...replay.log)
 
-    if (replay.log.length < REPLAY.dataSize) {
+    if (replay.log.length === 0) {
       REPLAY.final = true
     }
     return replay
@@ -340,10 +337,6 @@ export async function main(
       Game.setGame(gameObject.id, gameObject)
       TIME = () => Time.timestamp()
     } else if (MODE === MODE_REPLAY) {
-      REPLAY.logPointer = 0
-      REPLAY.dataSize = 10000
-      REPLAY.speeds = [0.5, 1, 2, 5, 10, 20, 50, 100, 250, 500]
-      REPLAY.speedIdx = 1
       const replay: ReplayData = await queryNextReplayBatch(gameId)
       if (!replay.game) {
         throw '[ 2021-05-29 no game received ]'
@@ -354,8 +347,6 @@ export async function main(
       REPLAY.lastRealTs = Time.timestamp()
       REPLAY.gameStartTs = parseInt(replay.log[0][4], 10)
       REPLAY.lastGameTs = REPLAY.gameStartTs
-      REPLAY.paused = false
-      REPLAY.skipNonActionPhases = false
 
       TIME = () => REPLAY.lastGameTs
     } else {
@@ -493,6 +484,10 @@ export async function main(
     doSetSpeedStatus()
   }
 
+  const replayOnSkipToggle = () => {
+    REPLAY.skipNonActionPhases = !REPLAY.skipNonActionPhases
+  }
+
   const intervals: NodeJS.Timeout[] = []
   let to: NodeJS.Timeout
   const clearIntervals = () => {
@@ -520,9 +515,6 @@ export async function main(
     doSetSpeedStatus()
   }
 
-  // // TODO: remove (make changable via interface)
-  // REPLAY.skipNonActionPhases = true
-
   if (MODE === MODE_PLAY) {
     Communication.onServerChange((msg: ServerEvent) => {
       const msgType = msg[0]
@@ -608,10 +600,8 @@ export async function main(
         const nextTs: Timestamp = REPLAY.gameStartTs + nextLogEntry[nextLogEntry.length - 1]
         if (nextTs > maxGameTs) {
           // next log entry is too far into the future
-          if (REPLAY.skipNonActionPhases && (maxGameTs + 50 < nextTs)) {
+          if (REPLAY.skipNonActionPhases && (maxGameTs + 500 * Time.MS < nextTs)) {
             const skipInterval = nextTs - currTs
-            // lets skip to the next log entry
-            // log.info('skipping non-action, from', maxGameTs, skipInterval)
             maxGameTs += skipInterval
           }
           break
@@ -874,6 +864,7 @@ export async function main(
     replayOnSpeedUp,
     replayOnSpeedDown,
     replayOnPauseToggle,
+    replayOnSkipToggle,
     previewImageUrl,
     player: {
       background: playerBgColor(),

src/frontend/views/Replay.vue

@@ -12,6 +12,12 @@
     >
       <div>
         <div>{{replayText}}</div>
+        <div>
+          <label>Skip no action phases: <input
+            type="checkbox"
+            v-model="skipNoAction"
+            @change="g.replayOnSkipToggle()" /></label>
+        </div>
         <button class="btn" @click="g.replayOnSpeedUp()">⏫</button>
         <button class="btn" @click="g.replayOnSpeedDown()">⏬</button>
         <button class="btn" @click="g.replayOnPauseToggle()">⏸️</button>
@@ -59,6 +65,7 @@ export default defineComponent({
       duration: 0,
       piecesDone: 0,
       piecesTotal: 0,
+      skipNoAction: true,
 
       overlay: '',
 
@@ -80,6 +87,7 @@ export default defineComponent({
         replayOnSpeedUp: () => {},
         replayOnSpeedDown: () => {},
         replayOnPauseToggle: () => {},
+        replayOnSkipToggle: () => {},
         connect: () => {},
         disconnect: () => {},
         unload: () => {},

src/server/GameLog.ts

@@ -8,6 +8,7 @@ import { DATA_DIR } from './../server/Dirs'
 
 const log = logger('GameLog.js')
 
+const LINES_PER_LOG_FILE = 10000
 const POST_GAME_LOG_DURATION = 5 * Time.MIN
 
 const shouldLog = (finishTs: Timestamp, currentTs: Timestamp): boolean => {
@@ -22,62 +23,63 @@ const shouldLog = (finishTs: Timestamp, currentTs: Timestamp): boolean => {
   return timeSinceGameEnd <= POST_GAME_LOG_DURATION
 }
 
-const filename = (gameId: string) => `${DATA_DIR}/log_${gameId}.log`
+const filename = (gameId: string, offset: number) => `${DATA_DIR}/log_${gameId}-${offset}.log`
+const idxname = (gameId: string) => `${DATA_DIR}/log_${gameId}.idx.log`
 
 const create = (gameId: string): void => {
-  const file = filename(gameId)
+  const idxfile = idxname(gameId)
-  if (!fs.existsSync(file)) {
+  if (!fs.existsSync(idxfile)) {
-    fs.appendFileSync(file, '')
+    const logfile = filename(gameId, 0)
+    fs.appendFileSync(logfile, "")
+    fs.appendFileSync(idxfile, JSON.stringify({
+      total: 0,
+      currentFile: logfile,
+      perFile: LINES_PER_LOG_FILE,
+    }))
   }
 }
 
 const exists = (gameId: string): boolean => {
-  const file = filename(gameId)
+  const idxfile = idxname(gameId)
-  return fs.existsSync(file)
+  return fs.existsSync(idxfile)
 }
 
 const _log = (gameId: string, ...args: Array<any>): void => {
-  const file = filename(gameId)
+  const idxfile = idxname(gameId)
-  if (!fs.existsSync(file)) {
+  if (!fs.existsSync(idxfile)) {
     return
   }
-  const str = JSON.stringify(args)
+
-  fs.appendFileSync(file, str + "\n")
+  const idx = JSON.parse(fs.readFileSync(idxfile, 'utf-8'))
+  idx.total++
+  fs.appendFileSync(idx.currentFile, JSON.stringify(args) + "\n")
+
+  // prepare next log file
+  if (idx.total % idx.perFile === 0) {
+    const logfile = filename(gameId, idx.total)
+    fs.appendFileSync(logfile, "")
+    idx.currentFile = logfile
+  }
+  fs.writeFileSync(idxfile, JSON.stringify(idx))
 }
 
-const get = async (
+const get = (
   gameId: string,
   offset: number = 0,
-  size: number = 10000
+): any[] => {
-): Promise<any[]> => {
+  const idxfile = idxname(gameId)
-  const file = filename(gameId)
+  if (!fs.existsSync(idxfile)) {
+    return []
+  }
+
+  const file = filename(gameId, offset)
   if (!fs.existsSync(file)) {
     return []
   }
-  return new Promise((resolve) => {
+
-    const instream = fs.createReadStream(file)
+  const log = fs.readFileSync(file, 'utf-8').split("\n")
-    const outstream = new stream.Writable()
+  return log.map(line => {
-    const rl = readline.createInterface(instream, outstream)
+    return JSON.parse(line)
-    const lines: any[] = []
-    let i = -1
-    rl.on('line', (line) => {
-      if (!line) {
-        // skip empty
-        return
-      }
-      i++
-      if (offset > i) {
-        return
-      }
-      if (offset + size <= i) {
-        rl.close()
-        return
-      }
-      lines.push(JSON.parse(line))
-    })
-    rl.on('close', () => {
-      resolve(lines)
-    })
   })
 }
 

src/server/main.ts

@@ -80,7 +80,7 @@ app.get('/api/replay-data', async (req, res): Promise<void> => {
     res.status(404).send({ reason: 'no log found' })
     return
   }
-  const log = await GameLog.get(gameId, offset, size)
+  const log = GameLog.get(gameId, offset)
   let game: GameType|null = null
   if (offset === 0) {
     // also need the game