split logs so that replay works for long games
This commit is contained in:
Zutatensuppe
parent
22f5ce0065
commit
514b3c6b22
10 changed files with 171 additions and 102 deletions
File diff suppressed because one or more lines are too long
1
build/public/assets/index.ab1d6e0f.js
Normal file
1
build/public/assets/index.ab1d6e0f.js
Normal file
File diff suppressed because one or more lines are too long
|
|
@ -4,7 +4,7 @@
|
|||
<meta charset="UTF-8">
|
||||
|
||||
<title>🧩 jigsaw.hyottoko.club</title>
|
||||
<script type="module" crossorigin src="/assets/index.7efa4c6c.js"></script>
|
||||
<script type="module" crossorigin src="/assets/index.ab1d6e0f.js"></script>
|
||||
<link rel="modulepreload" href="/assets/vendor.684f7bc8.js">
|
||||
<link rel="stylesheet" href="/assets/index.8f0efd0f.css">
|
||||
</head>
|
||||
|
|
|
|||
|
|
@ -3,8 +3,6 @@ import express from 'express';
|
|||
import compression from 'compression';
|
||||
import multer from 'multer';
|
||||
import fs from 'fs';
|
||||
import readline from 'readline';
|
||||
import stream from 'stream';
|
||||
import { fileURLToPath } from 'url';
|
||||
import { dirname } from 'path';
|
||||
import sizeOf from 'image-size';
|
||||
|
|
@ -1249,6 +1247,7 @@ const PUBLIC_DIR = `${BASE_DIR}/build/public/`;
|
|||
const DB_PATCHES_DIR = `${BASE_DIR}/src/dbpatches`;
|
||||
const DB_FILE = `${BASE_DIR}/data/db.sqlite`;
|
||||
|
||||
const LINES_PER_LOG_FILE = 10000;
|
||||
const POST_GAME_LOG_DURATION = 5 * Time.MIN;
|
||||
const shouldLog = (finishTs, currentTs) => {
|
||||
// when not finished yet, always log
|
||||
|
|
@ -1260,54 +1259,52 @@ const shouldLog = (finishTs, currentTs) => {
|
|||
const timeSinceGameEnd = currentTs - finishTs;
|
||||
return timeSinceGameEnd <= POST_GAME_LOG_DURATION;
|
||||
};
|
||||
const filename = (gameId) => `${DATA_DIR}/log_${gameId}.log`;
|
||||
const filename = (gameId, offset) => `${DATA_DIR}/log_${gameId}-${offset}.log`;
|
||||
const idxname = (gameId) => `${DATA_DIR}/log_${gameId}.idx.log`;
|
||||
const create = (gameId) => {
|
||||
const file = filename(gameId);
|
||||
if (!fs.existsSync(file)) {
|
||||
fs.appendFileSync(file, '');
|
||||
const idxfile = idxname(gameId);
|
||||
if (!fs.existsSync(idxfile)) {
|
||||
const logfile = filename(gameId, 0);
|
||||
fs.appendFileSync(logfile, "");
|
||||
fs.appendFileSync(idxfile, JSON.stringify({
|
||||
total: 0,
|
||||
currentFile: logfile,
|
||||
perFile: LINES_PER_LOG_FILE,
|
||||
}));
|
||||
}
|
||||
};
|
||||
const exists = (gameId) => {
|
||||
const file = filename(gameId);
|
||||
return fs.existsSync(file);
|
||||
const idxfile = idxname(gameId);
|
||||
return fs.existsSync(idxfile);
|
||||
};
|
||||
const _log = (gameId, ...args) => {
|
||||
const file = filename(gameId);
|
||||
if (!fs.existsSync(file)) {
|
||||
const idxfile = idxname(gameId);
|
||||
if (!fs.existsSync(idxfile)) {
|
||||
return;
|
||||
}
|
||||
const str = JSON.stringify(args);
|
||||
fs.appendFileSync(file, str + "\n");
|
||||
const idx = JSON.parse(fs.readFileSync(idxfile, 'utf-8'));
|
||||
idx.total++;
|
||||
fs.appendFileSync(idx.currentFile, JSON.stringify(args) + "\n");
|
||||
// prepare next log file
|
||||
if (idx.total % idx.perFile === 0) {
|
||||
const logfile = filename(gameId, idx.total);
|
||||
fs.appendFileSync(logfile, "");
|
||||
idx.currentFile = logfile;
|
||||
}
|
||||
fs.writeFileSync(idxfile, JSON.stringify(idx));
|
||||
};
|
||||
const get = async (gameId, offset = 0, size = 10000) => {
|
||||
const file = filename(gameId);
|
||||
const get = (gameId, offset = 0) => {
|
||||
const idxfile = idxname(gameId);
|
||||
if (!fs.existsSync(idxfile)) {
|
||||
return [];
|
||||
}
|
||||
const file = filename(gameId, offset);
|
||||
if (!fs.existsSync(file)) {
|
||||
return [];
|
||||
}
|
||||
return new Promise((resolve) => {
|
||||
const instream = fs.createReadStream(file);
|
||||
const outstream = new stream.Writable();
|
||||
const rl = readline.createInterface(instream, outstream);
|
||||
const lines = [];
|
||||
let i = -1;
|
||||
rl.on('line', (line) => {
|
||||
if (!line) {
|
||||
// skip empty
|
||||
return;
|
||||
}
|
||||
i++;
|
||||
if (offset > i) {
|
||||
return;
|
||||
}
|
||||
if (offset + size <= i) {
|
||||
rl.close();
|
||||
return;
|
||||
}
|
||||
lines.push(JSON.parse(line));
|
||||
});
|
||||
rl.on('close', () => {
|
||||
resolve(lines);
|
||||
});
|
||||
const log = fs.readFileSync(file, 'utf-8').split("\n");
|
||||
return log.map(line => {
|
||||
return JSON.parse(line);
|
||||
});
|
||||
};
|
||||
var GameLog = {
|
||||
|
|
@ -2061,7 +2058,7 @@ app.get('/api/replay-data', async (req, res) => {
|
|||
res.status(404).send({ reason: 'no log found' });
|
||||
return;
|
||||
}
|
||||
const log = await GameLog.get(gameId, offset, size);
|
||||
const log = GameLog.get(gameId, offset);
|
||||
let game = null;
|
||||
if (offset === 0) {
|
||||
// also need the game
|
||||
|
|
|
|||
72
scripts/split_logs.ts
Normal file
72
scripts/split_logs.ts
Normal file
|
|
@ -0,0 +1,72 @@
|
|||
import fs from 'fs'
|
||||
import readline from 'readline'
|
||||
import stream from 'stream'
|
||||
import { logger } from '../src/common/Util'
|
||||
import { DATA_DIR } from '../src/server/Dirs'
|
||||
|
||||
const log = logger('rewrite_logs')
|
||||
|
||||
const doit = (file: string): Promise<void> => {
|
||||
const filename = (offset: number) => file.replace(/\.log$/, `-${offset}.log`)
|
||||
const idxname = () => file.replace(/\.log$/, `.idx.log`)
|
||||
|
||||
let perfile = 10000
|
||||
const idx = {
|
||||
total: 0,
|
||||
currentFile: '',
|
||||
perFile: perfile,
|
||||
}
|
||||
|
||||
return new Promise((resolve) => {
|
||||
const instream = fs.createReadStream(DATA_DIR + '/' + file)
|
||||
const outstream = new stream.Writable()
|
||||
const rl = readline.createInterface(instream, outstream)
|
||||
|
||||
|
||||
let lines: any[] = []
|
||||
let offset = 0
|
||||
let count = 0
|
||||
rl.on('line', (line) => {
|
||||
if (!line) {
|
||||
// skip empty
|
||||
return
|
||||
}
|
||||
count++
|
||||
lines.push(line)
|
||||
if (count >= perfile) {
|
||||
const fn = filename(offset)
|
||||
idx.currentFile = fn
|
||||
idx.total += count
|
||||
fs.writeFileSync(DATA_DIR + '/' + fn, lines.join("\n"))
|
||||
count = 0
|
||||
offset += perfile
|
||||
lines = []
|
||||
}
|
||||
})
|
||||
|
||||
rl.on('close', () => {
|
||||
if (count > 0) {
|
||||
const fn = filename(offset)
|
||||
idx.currentFile = fn
|
||||
idx.total += count
|
||||
fs.writeFileSync(DATA_DIR + '/' + fn, lines.join("\n"))
|
||||
count = 0
|
||||
offset += perfile
|
||||
lines = []
|
||||
}
|
||||
|
||||
fs.writeFileSync(DATA_DIR + '/' + idxname(), JSON.stringify(idx))
|
||||
resolve()
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
let logs = fs.readdirSync(DATA_DIR)
|
||||
.filter(f => f.toLowerCase().match(/^log_.*\.log$/))
|
||||
|
||||
|
||||
;(async () => {
|
||||
for (const file of logs) {
|
||||
await doit(file)
|
||||
}
|
||||
})()
|
||||
|
|
@ -114,10 +114,9 @@ function connect(
|
|||
|
||||
async function requestReplayData(
|
||||
gameId: string,
|
||||
offset: number,
|
||||
size: number
|
||||
offset: number
|
||||
): Promise<ReplayData> {
|
||||
const args = { gameId, offset, size }
|
||||
const args = { gameId, offset }
|
||||
const res = await fetch(`/api/replay-data${Util.asQueryArgs(args)}`)
|
||||
const json: ReplayData = await res.json()
|
||||
return json
|
||||
|
|
|
|||
|
|
@ -69,7 +69,6 @@ interface Replay {
|
|||
skipNonActionPhases: boolean
|
||||
//
|
||||
dataOffset: number
|
||||
dataSize: number
|
||||
}
|
||||
|
||||
const shouldDrawPiece = (piece: Piece) => {
|
||||
|
|
@ -301,9 +300,8 @@ export async function main(
|
|||
lastRealTs: 0,
|
||||
lastGameTs: 0,
|
||||
gameStartTs: 0,
|
||||
skipNonActionPhases: false,
|
||||
skipNonActionPhases: true,
|
||||
dataOffset: 0,
|
||||
dataSize: 10000,
|
||||
}
|
||||
|
||||
Communication.onConnectionStateChange((state) => {
|
||||
|
|
@ -314,11 +312,10 @@ export async function main(
|
|||
gameId: string
|
||||
): Promise<ReplayData> => {
|
||||
const offset = REPLAY.dataOffset
|
||||
REPLAY.dataOffset += REPLAY.dataSize
|
||||
REPLAY.dataOffset += 10000 // meh
|
||||
const replay: ReplayData = await Communication.requestReplayData(
|
||||
gameId,
|
||||
offset,
|
||||
REPLAY.dataSize
|
||||
offset
|
||||
)
|
||||
|
||||
// cut log that was already handled
|
||||
|
|
@ -326,7 +323,7 @@ export async function main(
|
|||
REPLAY.logPointer = 0
|
||||
REPLAY.log.push(...replay.log)
|
||||
|
||||
if (replay.log.length < REPLAY.dataSize) {
|
||||
if (replay.log.length === 0) {
|
||||
REPLAY.final = true
|
||||
}
|
||||
return replay
|
||||
|
|
@ -340,10 +337,6 @@ export async function main(
|
|||
Game.setGame(gameObject.id, gameObject)
|
||||
TIME = () => Time.timestamp()
|
||||
} else if (MODE === MODE_REPLAY) {
|
||||
REPLAY.logPointer = 0
|
||||
REPLAY.dataSize = 10000
|
||||
REPLAY.speeds = [0.5, 1, 2, 5, 10, 20, 50, 100, 250, 500]
|
||||
REPLAY.speedIdx = 1
|
||||
const replay: ReplayData = await queryNextReplayBatch(gameId)
|
||||
if (!replay.game) {
|
||||
throw '[ 2021-05-29 no game received ]'
|
||||
|
|
@ -354,8 +347,6 @@ export async function main(
|
|||
REPLAY.lastRealTs = Time.timestamp()
|
||||
REPLAY.gameStartTs = parseInt(replay.log[0][4], 10)
|
||||
REPLAY.lastGameTs = REPLAY.gameStartTs
|
||||
REPLAY.paused = false
|
||||
REPLAY.skipNonActionPhases = false
|
||||
|
||||
TIME = () => REPLAY.lastGameTs
|
||||
} else {
|
||||
|
|
@ -493,6 +484,10 @@ export async function main(
|
|||
doSetSpeedStatus()
|
||||
}
|
||||
|
||||
const replayOnSkipToggle = () => {
|
||||
REPLAY.skipNonActionPhases = !REPLAY.skipNonActionPhases
|
||||
}
|
||||
|
||||
const intervals: NodeJS.Timeout[] = []
|
||||
let to: NodeJS.Timeout
|
||||
const clearIntervals = () => {
|
||||
|
|
@ -520,9 +515,6 @@ export async function main(
|
|||
doSetSpeedStatus()
|
||||
}
|
||||
|
||||
// // TODO: remove (make changable via interface)
|
||||
// REPLAY.skipNonActionPhases = true
|
||||
|
||||
if (MODE === MODE_PLAY) {
|
||||
Communication.onServerChange((msg: ServerEvent) => {
|
||||
const msgType = msg[0]
|
||||
|
|
@ -608,10 +600,8 @@ export async function main(
|
|||
const nextTs: Timestamp = REPLAY.gameStartTs + nextLogEntry[nextLogEntry.length - 1]
|
||||
if (nextTs > maxGameTs) {
|
||||
// next log entry is too far into the future
|
||||
if (REPLAY.skipNonActionPhases && (maxGameTs + 50 < nextTs)) {
|
||||
if (REPLAY.skipNonActionPhases && (maxGameTs + 500 * Time.MS < nextTs)) {
|
||||
const skipInterval = nextTs - currTs
|
||||
// lets skip to the next log entry
|
||||
// log.info('skipping non-action, from', maxGameTs, skipInterval)
|
||||
maxGameTs += skipInterval
|
||||
}
|
||||
break
|
||||
|
|
@ -874,6 +864,7 @@ export async function main(
|
|||
replayOnSpeedUp,
|
||||
replayOnSpeedDown,
|
||||
replayOnPauseToggle,
|
||||
replayOnSkipToggle,
|
||||
previewImageUrl,
|
||||
player: {
|
||||
background: playerBgColor(),
|
||||
|
|
|
|||
|
|
@ -12,6 +12,12 @@
|
|||
>
|
||||
<div>
|
||||
<div>{{replayText}}</div>
|
||||
<div>
|
||||
<label>Skip no action phases: <input
|
||||
type="checkbox"
|
||||
v-model="skipNoAction"
|
||||
@change="g.replayOnSkipToggle()" /></label>
|
||||
</div>
|
||||
<button class="btn" @click="g.replayOnSpeedUp()">⏫</button>
|
||||
<button class="btn" @click="g.replayOnSpeedDown()">⏬</button>
|
||||
<button class="btn" @click="g.replayOnPauseToggle()">⏸️</button>
|
||||
|
|
@ -59,6 +65,7 @@ export default defineComponent({
|
|||
duration: 0,
|
||||
piecesDone: 0,
|
||||
piecesTotal: 0,
|
||||
skipNoAction: true,
|
||||
|
||||
overlay: '',
|
||||
|
||||
|
|
@ -80,6 +87,7 @@ export default defineComponent({
|
|||
replayOnSpeedUp: () => {},
|
||||
replayOnSpeedDown: () => {},
|
||||
replayOnPauseToggle: () => {},
|
||||
replayOnSkipToggle: () => {},
|
||||
connect: () => {},
|
||||
disconnect: () => {},
|
||||
unload: () => {},
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ import { DATA_DIR } from './../server/Dirs'
|
|||
|
||||
const log = logger('GameLog.js')
|
||||
|
||||
const LINES_PER_LOG_FILE = 10000
|
||||
const POST_GAME_LOG_DURATION = 5 * Time.MIN
|
||||
|
||||
const shouldLog = (finishTs: Timestamp, currentTs: Timestamp): boolean => {
|
||||
|
|
@ -22,62 +23,63 @@ const shouldLog = (finishTs: Timestamp, currentTs: Timestamp): boolean => {
|
|||
return timeSinceGameEnd <= POST_GAME_LOG_DURATION
|
||||
}
|
||||
|
||||
const filename = (gameId: string) => `${DATA_DIR}/log_${gameId}.log`
|
||||
const filename = (gameId: string, offset: number) => `${DATA_DIR}/log_${gameId}-${offset}.log`
|
||||
const idxname = (gameId: string) => `${DATA_DIR}/log_${gameId}.idx.log`
|
||||
|
||||
const create = (gameId: string): void => {
|
||||
const file = filename(gameId)
|
||||
if (!fs.existsSync(file)) {
|
||||
fs.appendFileSync(file, '')
|
||||
const idxfile = idxname(gameId)
|
||||
if (!fs.existsSync(idxfile)) {
|
||||
const logfile = filename(gameId, 0)
|
||||
fs.appendFileSync(logfile, "")
|
||||
fs.appendFileSync(idxfile, JSON.stringify({
|
||||
total: 0,
|
||||
currentFile: logfile,
|
||||
perFile: LINES_PER_LOG_FILE,
|
||||
}))
|
||||
}
|
||||
}
|
||||
|
||||
const exists = (gameId: string): boolean => {
|
||||
const file = filename(gameId)
|
||||
return fs.existsSync(file)
|
||||
const idxfile = idxname(gameId)
|
||||
return fs.existsSync(idxfile)
|
||||
}
|
||||
|
||||
const _log = (gameId: string, ...args: Array<any>): void => {
|
||||
const file = filename(gameId)
|
||||
if (!fs.existsSync(file)) {
|
||||
const idxfile = idxname(gameId)
|
||||
if (!fs.existsSync(idxfile)) {
|
||||
return
|
||||
}
|
||||
const str = JSON.stringify(args)
|
||||
fs.appendFileSync(file, str + "\n")
|
||||
|
||||
const idx = JSON.parse(fs.readFileSync(idxfile, 'utf-8'))
|
||||
idx.total++
|
||||
fs.appendFileSync(idx.currentFile, JSON.stringify(args) + "\n")
|
||||
|
||||
// prepare next log file
|
||||
if (idx.total % idx.perFile === 0) {
|
||||
const logfile = filename(gameId, idx.total)
|
||||
fs.appendFileSync(logfile, "")
|
||||
idx.currentFile = logfile
|
||||
}
|
||||
fs.writeFileSync(idxfile, JSON.stringify(idx))
|
||||
}
|
||||
|
||||
const get = async (
|
||||
const get = (
|
||||
gameId: string,
|
||||
offset: number = 0,
|
||||
size: number = 10000
|
||||
): Promise<any[]> => {
|
||||
const file = filename(gameId)
|
||||
): any[] => {
|
||||
const idxfile = idxname(gameId)
|
||||
if (!fs.existsSync(idxfile)) {
|
||||
return []
|
||||
}
|
||||
|
||||
const file = filename(gameId, offset)
|
||||
if (!fs.existsSync(file)) {
|
||||
return []
|
||||
}
|
||||
return new Promise((resolve) => {
|
||||
const instream = fs.createReadStream(file)
|
||||
const outstream = new stream.Writable()
|
||||
const rl = readline.createInterface(instream, outstream)
|
||||
const lines: any[] = []
|
||||
let i = -1
|
||||
rl.on('line', (line) => {
|
||||
if (!line) {
|
||||
// skip empty
|
||||
return
|
||||
}
|
||||
i++
|
||||
if (offset > i) {
|
||||
return
|
||||
}
|
||||
if (offset + size <= i) {
|
||||
rl.close()
|
||||
return
|
||||
}
|
||||
lines.push(JSON.parse(line))
|
||||
})
|
||||
rl.on('close', () => {
|
||||
resolve(lines)
|
||||
})
|
||||
|
||||
const log = fs.readFileSync(file, 'utf-8').split("\n")
|
||||
return log.map(line => {
|
||||
return JSON.parse(line)
|
||||
})
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -80,7 +80,7 @@ app.get('/api/replay-data', async (req, res): Promise<void> => {
|
|||
res.status(404).send({ reason: 'no log found' })
|
||||
return
|
||||
}
|
||||
const log = await GameLog.get(gameId, offset, size)
|
||||
const log = GameLog.get(gameId, offset)
|
||||
let game: GameType|null = null
|
||||
if (offset === 0) {
|
||||
// also need the game
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue