split logs so that replay works for long games

src/frontend/Communication.ts

@@ -114,10 +114,9 @@ function connect(
 
 async function requestReplayData(
   gameId: string,
-  offset: number,
-  size: number
+  offset: number
 ): Promise<ReplayData> {
-  const args = { gameId, offset, size }
+  const args = { gameId, offset }
   const res = await fetch(`/api/replay-data${Util.asQueryArgs(args)}`)
   const json: ReplayData = await res.json()
   return json

src/frontend/game.ts

@@ -69,7 +69,6 @@ interface Replay {
   skipNonActionPhases: boolean
   //
   dataOffset: number
-  dataSize: number
 }
 
 const shouldDrawPiece = (piece: Piece) => {
@@ -301,9 +300,8 @@ export async function main(
     lastRealTs: 0,
     lastGameTs: 0,
     gameStartTs: 0,
-    skipNonActionPhases: false,
+    skipNonActionPhases: true,
     dataOffset: 0,
-    dataSize: 10000,
   }
 
   Communication.onConnectionStateChange((state) => {
@@ -314,11 +312,10 @@ export async function main(
     gameId: string
   ): Promise<ReplayData> => {
     const offset = REPLAY.dataOffset
-    REPLAY.dataOffset += REPLAY.dataSize
+    REPLAY.dataOffset += 10000 // meh
     const replay: ReplayData = await Communication.requestReplayData(
       gameId,
-      offset,
-      REPLAY.dataSize
+      offset
     )
 
     // cut log that was already handled
@@ -326,7 +323,7 @@ export async function main(
     REPLAY.logPointer = 0
     REPLAY.log.push(...replay.log)
 
-    if (replay.log.length < REPLAY.dataSize) {
+    if (replay.log.length === 0) {
       REPLAY.final = true
     }
     return replay
@@ -340,10 +337,6 @@ export async function main(
       Game.setGame(gameObject.id, gameObject)
       TIME = () => Time.timestamp()
     } else if (MODE === MODE_REPLAY) {
-      REPLAY.logPointer = 0
-      REPLAY.dataSize = 10000
-      REPLAY.speeds = [0.5, 1, 2, 5, 10, 20, 50, 100, 250, 500]
-      REPLAY.speedIdx = 1
       const replay: ReplayData = await queryNextReplayBatch(gameId)
       if (!replay.game) {
         throw '[ 2021-05-29 no game received ]'
@@ -354,8 +347,6 @@ export async function main(
       REPLAY.lastRealTs = Time.timestamp()
       REPLAY.gameStartTs = parseInt(replay.log[0][4], 10)
       REPLAY.lastGameTs = REPLAY.gameStartTs
-      REPLAY.paused = false
-      REPLAY.skipNonActionPhases = false
 
       TIME = () => REPLAY.lastGameTs
     } else {
@@ -493,6 +484,10 @@ export async function main(
     doSetSpeedStatus()
   }
 
+  const replayOnSkipToggle = () => {
+    REPLAY.skipNonActionPhases = !REPLAY.skipNonActionPhases
+  }
+
   const intervals: NodeJS.Timeout[] = []
   let to: NodeJS.Timeout
   const clearIntervals = () => {
@@ -520,9 +515,6 @@ export async function main(
     doSetSpeedStatus()
   }
 
-  // // TODO: remove (make changable via interface)
-  // REPLAY.skipNonActionPhases = true
-
   if (MODE === MODE_PLAY) {
     Communication.onServerChange((msg: ServerEvent) => {
       const msgType = msg[0]
@@ -608,10 +600,8 @@ export async function main(
         const nextTs: Timestamp = REPLAY.gameStartTs + nextLogEntry[nextLogEntry.length - 1]
         if (nextTs > maxGameTs) {
           // next log entry is too far into the future
-          if (REPLAY.skipNonActionPhases && (maxGameTs + 50 < nextTs)) {
+          if (REPLAY.skipNonActionPhases && (maxGameTs + 500 * Time.MS < nextTs)) {
             const skipInterval = nextTs - currTs
-            // lets skip to the next log entry
-            // log.info('skipping non-action, from', maxGameTs, skipInterval)
             maxGameTs += skipInterval
           }
           break
@@ -874,6 +864,7 @@ export async function main(
     replayOnSpeedUp,
     replayOnSpeedDown,
     replayOnPauseToggle,
+    replayOnSkipToggle,
     previewImageUrl,
     player: {
       background: playerBgColor(),

src/frontend/views/Replay.vue

@@ -12,6 +12,12 @@
     >
       <div>
         <div>{{replayText}}</div>
+        <div>
+          <label>Skip no action phases: <input
+            type="checkbox"
+            v-model="skipNoAction"
+            @change="g.replayOnSkipToggle()" /></label>
+        </div>
         <button class="btn" @click="g.replayOnSpeedUp()">⏫</button>
         <button class="btn" @click="g.replayOnSpeedDown()">⏬</button>
         <button class="btn" @click="g.replayOnPauseToggle()">⏸️</button>
@@ -59,6 +65,7 @@ export default defineComponent({
       duration: 0,
       piecesDone: 0,
       piecesTotal: 0,
+      skipNoAction: true,
 
       overlay: '',
 
@@ -80,6 +87,7 @@ export default defineComponent({
         replayOnSpeedUp: () => {},
         replayOnSpeedDown: () => {},
         replayOnPauseToggle: () => {},
+        replayOnSkipToggle: () => {},
         connect: () => {},
         disconnect: () => {},
         unload: () => {},

src/server/GameLog.ts

@@ -8,6 +8,7 @@ import { DATA_DIR } from './../server/Dirs'
 
 const log = logger('GameLog.js')
 
+const LINES_PER_LOG_FILE = 10000
 const POST_GAME_LOG_DURATION = 5 * Time.MIN
 
 const shouldLog = (finishTs: Timestamp, currentTs: Timestamp): boolean => {
@@ -22,62 +23,63 @@ const shouldLog = (finishTs: Timestamp, currentTs: Timestamp): boolean => {
   return timeSinceGameEnd <= POST_GAME_LOG_DURATION
 }
 
-const filename = (gameId: string) => `${DATA_DIR}/log_${gameId}.log`
+const filename = (gameId: string, offset: number) => `${DATA_DIR}/log_${gameId}-${offset}.log`
+const idxname = (gameId: string) => `${DATA_DIR}/log_${gameId}.idx.log`
 
 const create = (gameId: string): void => {
-  const file = filename(gameId)
-  if (!fs.existsSync(file)) {
-    fs.appendFileSync(file, '')
+  const idxfile = idxname(gameId)
+  if (!fs.existsSync(idxfile)) {
+    const logfile = filename(gameId, 0)
+    fs.appendFileSync(logfile, "")
+    fs.appendFileSync(idxfile, JSON.stringify({
+      total: 0,
+      currentFile: logfile,
+      perFile: LINES_PER_LOG_FILE,
+    }))
   }
 }
 
 const exists = (gameId: string): boolean => {
-  const file = filename(gameId)
-  return fs.existsSync(file)
+  const idxfile = idxname(gameId)
+  return fs.existsSync(idxfile)
 }
 
 const _log = (gameId: string, ...args: Array<any>): void => {
-  const file = filename(gameId)
-  if (!fs.existsSync(file)) {
+  const idxfile = idxname(gameId)
+  if (!fs.existsSync(idxfile)) {
     return
   }
-  const str = JSON.stringify(args)
-  fs.appendFileSync(file, str + "\n")
+
+  const idx = JSON.parse(fs.readFileSync(idxfile, 'utf-8'))
+  idx.total++
+  fs.appendFileSync(idx.currentFile, JSON.stringify(args) + "\n")
+
+  // prepare next log file
+  if (idx.total % idx.perFile === 0) {
+    const logfile = filename(gameId, idx.total)
+    fs.appendFileSync(logfile, "")
+    idx.currentFile = logfile
+  }
+  fs.writeFileSync(idxfile, JSON.stringify(idx))
 }
 
-const get = async (
+const get = (
   gameId: string,
   offset: number = 0,
-  size: number = 10000
-): Promise<any[]> => {
-  const file = filename(gameId)
+): any[] => {
+  const idxfile = idxname(gameId)
+  if (!fs.existsSync(idxfile)) {
+    return []
+  }
+
+  const file = filename(gameId, offset)
   if (!fs.existsSync(file)) {
     return []
   }
-  return new Promise((resolve) => {
-    const instream = fs.createReadStream(file)
-    const outstream = new stream.Writable()
-    const rl = readline.createInterface(instream, outstream)
-    const lines: any[] = []
-    let i = -1
-    rl.on('line', (line) => {
-      if (!line) {
-        // skip empty
-        return
-      }
-      i++
-      if (offset > i) {
-        return
-      }
-      if (offset + size <= i) {
-        rl.close()
-        return
-      }
-      lines.push(JSON.parse(line))
-    })
-    rl.on('close', () => {
-      resolve(lines)
-    })
+
+  const log = fs.readFileSync(file, 'utf-8').split("\n")
+  return log.map(line => {
+    return JSON.parse(line)
   })
 }
 

src/server/main.ts

@@ -80,7 +80,7 @@ app.get('/api/replay-data', async (req, res): Promise<void> => {
     res.status(404).send({ reason: 'no log found' })
     return
   }
-  const log = await GameLog.get(gameId, offset, size)
+  const log = GameLog.get(gameId, offset)
   let game: GameType|null = null
   if (offset === 0) {
     // also need the game