smaller logs

scripts/rewrite_logs.ts

@@ -1,82 +0,0 @@
-import fs from 'fs'
-import Protocol from '../src/common/Protocol'
-import { logger } from '../src/common/Util'
-import { DATA_DIR } from '../src/server/Dirs'
-
-const log = logger('rewrite_logs')
-
-const filename = (gameId) => `${DATA_DIR}/log_${gameId}.log`
-
-const rewrite = (gameId) => {
-  const file = filename(gameId)
-  log.log(file)
-  if (!fs.existsSync(file)) {
-    return []
-  }
-  let playerIds = [];
-  let startTs = null
-  const lines = fs.readFileSync(file, 'utf-8').split("\n")
-  const linesNew = lines.filter(line => !!line).map((line) => {
-    const json = JSON.parse(line)
-    const m = {
-      createGame: Protocol.LOG_HEADER,
-      addPlayer: Protocol.LOG_ADD_PLAYER,
-      handleInput: Protocol.LOG_HANDLE_INPUT,
-    }
-    const action = json[0]
-    if (action in m) {
-      json[0] = m[action]
-      if (json[0] === Protocol.LOG_HANDLE_INPUT) {
-        const inputm = {
-          down: Protocol.INPUT_EV_MOUSE_DOWN,
-          up: Protocol.INPUT_EV_MOUSE_UP,
-          move: Protocol.INPUT_EV_MOUSE_MOVE,
-          zoomin: Protocol.INPUT_EV_ZOOM_IN,
-          zoomout: Protocol.INPUT_EV_ZOOM_OUT,
-          bg_color: Protocol.INPUT_EV_BG_COLOR,
-          player_color: Protocol.INPUT_EV_PLAYER_COLOR,
-          player_name: Protocol.INPUT_EV_PLAYER_NAME,
-        }
-        const inputa = json[2][0]
-        if (inputa in inputm) {
-          json[2][0] = inputm[inputa]
-        } else {
-          throw '[ invalid input log line: "' + line + '" ]'
-        }
-      }
-    } else {
-      throw '[ invalid general log line: "' + line + '" ]'
-    }
-
-    if (json[0] === Protocol.LOG_ADD_PLAYER) {
-      if (playerIds.indexOf(json[1]) === -1) {
-        playerIds.push(json[1])
-      } else {
-        json[0] = Protocol.LOG_UPDATE_PLAYER
-        json[1] = playerIds.indexOf(json[1])
-      }
-    }
-
-    if (json[0] === Protocol.LOG_HANDLE_INPUT) {
-      json[1] = playerIds.indexOf(json[1])
-      if (json[1] === -1) {
-        throw '[ invalid player ... "' + line + '" ]'
-      }
-    }
-
-    if (json[0] === Protocol.LOG_HEADER) {
-      startTs = json[json.length - 1]
-      json[4] = json[3]
-      json[3] = json[2]
-      json[2] = json[1]
-      json[1] = 1
-    } else {
-      json[json.length - 1] = json[json.length - 1] - startTs
-    }
-    return JSON.stringify(json)
-  })
-
-  fs.writeFileSync(file, linesNew.join("\n") + "\n")
-}
-
-rewrite(process.argv[2])

scripts/split_logs.ts

@@ -1,72 +1,73 @@
 import fs from 'fs'
-import readline from 'readline'
-import stream from 'stream'
 import { logger } from '../src/common/Util'
 import { DATA_DIR } from '../src/server/Dirs'
+import { filename } from '../src/server/GameLog'
 
 const log = logger('rewrite_logs')
 
-const doit = (file: string): Promise<void> => {
-  const filename = (offset: number) => file.replace(/\.log$/, `-${offset}.log`)
-  const idxname = () => file.replace(/\.log$/, `.idx.log`)
-
-  let perfile = 10000
-  const idx = {
-    total: 0,
-    currentFile: '',
-    perFile: perfile,
-  }
-
-  return new Promise((resolve) => {
-    const instream = fs.createReadStream(DATA_DIR + '/' + file)
-    const outstream = new stream.Writable()
-    const rl = readline.createInterface(instream, outstream)
-
-
-    let lines: any[] = []
-    let offset = 0
-    let count = 0
-    rl.on('line', (line) => {
-      if (!line) {
-        // skip empty
-        return
-      }
-      count++
-      lines.push(line)
-      if (count >= perfile) {
-        const fn = filename(offset)
-        idx.currentFile = fn
-        idx.total += count
-        fs.writeFileSync(DATA_DIR + '/' + fn, lines.join("\n"))
-        count = 0
-        offset += perfile
-        lines = []
-      }
-    })
-
-    rl.on('close', () => {
-      if (count > 0) {
-        const fn = filename(offset)
-        idx.currentFile = fn
-        idx.total += count
-        fs.writeFileSync(DATA_DIR + '/' + fn, lines.join("\n"))
-        count = 0
-        offset += perfile
-        lines = []
-      }
-
-      fs.writeFileSync(DATA_DIR + '/' + idxname(), JSON.stringify(idx))
-      resolve()
-    })
-  })
+interface IdxOld {
+  total: number
+  currentFile: string
+  perFile: number
 }
 
-let logs = fs.readdirSync(DATA_DIR)
-  .filter(f => f.toLowerCase().match(/^log_.*\.log$/))
+interface Idx {
+  gameId: string
+  total: number
+  lastTs: number
+  currentFile: string
+  perFile: number
+}
+const doit = (idxfile: string): void => {
+  const gameId: string = (idxfile.match(/^log_([a-z0-9]+)\.idx\.log$/) as any[])[1]
+  const idxOld: IdxOld = JSON.parse(fs.readFileSync(DATA_DIR + '/' + idxfile, 'utf-8'))
+
+  let currentFile = filename(gameId, 0)
+  const idxNew: Idx = {
+    gameId: gameId,
+    total: 0,
+    lastTs: 0,
+    currentFile: currentFile,
+    perFile: idxOld.perFile
+  }
+
+  let firstTs = 0
+  while (fs.existsSync(currentFile)) {
+    idxNew.currentFile = currentFile
+    const log = fs.readFileSync(currentFile, 'utf-8').split("\n")
+    const newLines = []
+    const lines = log.filter(line => !!line).map(line => {
+      return JSON.parse(line)
+    })
+    for (const l of lines) {
+      if (idxNew.total === 0) {
+        firstTs = l[4]
+        idxNew.lastTs = l[4]
+        newLines.push(JSON.stringify(l).slice(1, -1))
+      } else {
+        const ts = firstTs + l[l.length - 1]
+        const diff = ts - idxNew.lastTs
+        idxNew.lastTs = ts
+        const newL = l.slice(0, -1)
+        newL.push(diff)
+        newLines.push(JSON.stringify(newL).slice(1, -1))
+      }
+      idxNew.total++
+    }
+    fs.writeFileSync(idxNew.currentFile, newLines.join("\n") + "\n")
+    currentFile = filename(gameId, idxNew.total)
+  }
+
+  fs.writeFileSync(DATA_DIR + '/' + idxfile, JSON.stringify(idxNew))
+  console.log('done: ' + gameId)
+}
+
+let indexfiles = fs.readdirSync(DATA_DIR)
+  .filter(f => f.toLowerCase().match(/^log_[a-z0-9]+\.idx\.log$/))
 
 
 ;(async () => {
-  for (const file of logs) {
+  for (const file of indexfiles) {
     await doit(file)
   }
 })()