diff --git a/common/GameCommon.js b/common/GameCommon.js
index c2ece7c..92a1318 100644
--- a/common/GameCommon.js
+++ b/common/GameCommon.js
@@ -1,4 +1,5 @@
 import Geometry from './Geometry.js'
+import Protocol from './Protocol.js'
 import Util from './Util.js'
 
 const GAMES = {}
@@ -42,12 +43,36 @@ function setGame(gameId, game) {
   GAMES[gameId] = game
 }
 
+function getPlayerIndexById(gameId, playerId) {
+  let i = 0;
+  for (let player of GAMES[gameId].players) {
+    if (Util.decodePlayer(player).id === playerId) {
+      return i
+    }
+    i++
+  }
+  return -1
+}
+
+function getPlayerIdByIndex(gameId, playerIndex) {
+  if (GAMES[gameId].players.length > playerIndex) {
+    return Util.decodePlayer(GAMES[gameId].players[playerIndex]).id
+  }
+  return null
+}
+
 function getPlayer(gameId, playerId) {
-  return Util.decodePlayer(GAMES[gameId].players[playerId])
+  let idx = getPlayerIndexById(gameId, playerId)
+  return Util.decodePlayer(GAMES[gameId].players[idx])
 }
 
 function setPlayer(gameId, playerId, player) {
-  GAMES[gameId].players[playerId] = Util.encodePlayer(player)
+  let idx = getPlayerIndexById(gameId, playerId)
+  if (idx === -1) {
+    GAMES[gameId].players.push(Util.encodePlayer(player))
+  } else {
+    GAMES[gameId].players[idx] = Util.encodePlayer(player)
+  }
 }
 
 function setTile(gameId, tileIdx, tile) {
@@ -59,7 +84,8 @@ function setPuzzleData(gameId, data) {
 }
 
 function playerExists(gameId, playerId) {
-  return !!GAMES[gameId].players[playerId]
+  const idx = getPlayerIndexById(gameId, playerId)
+  return idx !== -1
 }
 
 function getRelevantPlayers(gameId, ts) {
@@ -77,7 +103,7 @@ function getActivePlayers(gameId, ts) {
 }
 
 function addPlayer(gameId, playerId, ts) {
-  if (!GAMES[gameId].players[playerId]) {
+  if (!playerExists(gameId, playerId)) {
     setPlayer(gameId, playerId, __createPlayerObject(playerId, ts))
   } else {
     changePlayer(gameId, playerId, { ts })
@@ -118,7 +144,7 @@ function getAllGames() {
 
 function getAllPlayers(gameId) {
   return GAMES[gameId]
-    ? Object.values(GAMES[gameId].players).map(Util.decodePlayer)
+    ? GAMES[gameId].players.map(Util.decodePlayer)
     : []
 }
 
@@ -436,7 +462,7 @@ function handleInput(gameId, playerId, input, ts) {
   }
 
   const _playerChange = () => {
-    changes.push(['player', GAMES[gameId].players[playerId]])
+    changes.push(['player', Util.encodePlayer(getPlayer(gameId, playerId))])
   }
 
   // put both tiles (and their grouped tiles) in the same group
@@ -482,19 +508,19 @@ function handleInput(gameId, playerId, input, ts) {
   }
 
   const type = input[0]
-  if (type === 'bg_color') {
+  if (type === Protocol.INPUT_EV_BG_COLOR) {
     const bgcolor = input[1]
     changePlayer(gameId, playerId, { bgcolor, ts })
     _playerChange()
-  } else if (type === 'player_color') {
+  } else if (type === Protocol.INPUT_EV_PLAYER_COLOR) {
     const color = input[1]
     changePlayer(gameId, playerId, { color, ts })
     _playerChange()
-  } else if (type === 'player_name') {
+  } else if (type === Protocol.INPUT_EV_PLAYER_NAME) {
     const name = `${input[1]}`.substr(0, 16)
     changePlayer(gameId, playerId, { name, ts })
     _playerChange()
-  } else if (type === 'down') {
+  } else if (type === Protocol.INPUT_EV_MOUSE_DOWN) {
     const x = input[1]
     const y = input[2]
     const pos = {x, y}
@@ -515,7 +541,7 @@ function handleInput(gameId, playerId, input, ts) {
     }
     evtInfo._last_mouse = pos
 
-  } else if (type === 'move') {
+  } else if (type === Protocol.INPUT_EV_MOUSE_MOVE) {
     const x = input[1]
     const y = input[2]
     const pos = {x, y}
@@ -538,7 +564,7 @@ function handleInput(gameId, playerId, input, ts) {
     }
     evtInfo._last_mouse = pos
 
-  } else if (type === 'up') {
+  } else if (type === Protocol.INPUT_EV_MOUSE_UP) {
     const x = input[1]
     const y = input[2]
     const pos = {x, y}
@@ -641,6 +667,8 @@ export default {
   getPlayerBgColor,
   getPlayerColor,
   getPlayerName,
+  getPlayerIndexById,
+  getPlayerIdByIndex,
   changePlayer,
   setPlayer,
   setTile,
diff --git a/common/Protocol.js b/common/Protocol.js
index cd9132f..297b7fa 100644
--- a/common/Protocol.js
+++ b/common/Protocol.js
@@ -45,6 +45,20 @@ const EV_CLIENT_EVENT = 2
 const EV_CLIENT_INIT = 3
 const EV_CLIENT_INIT_REPLAY = 6
 
+const LOG_HEADER = 1
+const LOG_ADD_PLAYER = 2
+const LOG_UPDATE_PLAYER = 4
+const LOG_HANDLE_INPUT = 3
+
+const INPUT_EV_MOUSE_DOWN = 1
+const INPUT_EV_MOUSE_UP = 2
+const INPUT_EV_MOUSE_MOVE = 3
+const INPUT_EV_ZOOM_IN = 4
+const INPUT_EV_ZOOM_OUT = 5
+const INPUT_EV_BG_COLOR = 6
+const INPUT_EV_PLAYER_COLOR = 7
+const INPUT_EV_PLAYER_NAME = 8
+
 export default {
   EV_SERVER_EVENT,
   EV_SERVER_INIT,
@@ -52,4 +66,18 @@ export default {
   EV_CLIENT_EVENT,
   EV_CLIENT_INIT,
   EV_CLIENT_INIT_REPLAY,
+
+  LOG_HEADER,
+  LOG_ADD_PLAYER,
+  LOG_UPDATE_PLAYER,
+  LOG_HANDLE_INPUT,
+
+  INPUT_EV_MOUSE_DOWN,
+  INPUT_EV_MOUSE_UP,
+  INPUT_EV_MOUSE_MOVE,
+  INPUT_EV_ZOOM_IN,
+  INPUT_EV_ZOOM_OUT,
+  INPUT_EV_BG_COLOR,
+  INPUT_EV_PLAYER_COLOR,
+  INPUT_EV_PLAYER_NAME,
 }
diff --git a/game/Game.js b/game/Game.js
index f0b21c5..afae242 100644
--- a/game/Game.js
+++ b/game/Game.js
@@ -6,6 +6,7 @@ export default {
   getActivePlayers: GameCommon.getActivePlayers,
   addPlayer: GameCommon.addPlayer,
   handleInput: GameCommon.handleInput,
+  getPlayerIdByIndex: GameCommon.getPlayerIdByIndex,
   getPlayerBgColor: GameCommon.getPlayerBgColor,
   getPlayerColor: GameCommon.getPlayerColor,
   getPlayerName: GameCommon.getPlayerName,
diff --git a/game/game.js b/game/game.js
index ddceb1a..ffcdd8c 100644
--- a/game/game.js
+++ b/game/game.js
@@ -9,6 +9,7 @@ import PuzzleGraphics from './PuzzleGraphics.js'
 import Game from './Game.js'
 import fireworksController from './Fireworks.js'
 import { Rng } from '../common/Rng.js'
+import Protocol from '../common/Protocol.js'
 
 if (typeof GAME_ID === 'undefined') throw '[ GAME_ID not set ]'
 if (typeof WS_ADDRESS === 'undefined') throw '[ WS_ADDRESS not set ]'
@@ -298,7 +299,7 @@ export default class EventAdapter {
         x: e.offsetX,
         y: e.offsetY,
       })
-      this.addEvent(['down', pos.x, pos.y])
+      this.addEvent([Protocol.INPUT_EV_MOUSE_DOWN, pos.x, pos.y])
     }
   }
 
@@ -308,7 +309,7 @@ export default class EventAdapter {
         x: e.offsetX,
         y: e.offsetY,
       })
-      this.addEvent(['up', pos.x, pos.y])
+      this.addEvent([Protocol.INPUT_EV_MOUSE_UP, pos.x, pos.y])
     }
   }
 
@@ -317,7 +318,7 @@ export default class EventAdapter {
       x: e.offsetX,
       y: e.offsetY,
     })
-    this.addEvent(['move', pos.x, pos.y])
+    this.addEvent([Protocol.INPUT_EV_MOUSE_MOVE, pos.x, pos.y])
   }
 
   _wheel(e) {
@@ -325,7 +326,7 @@ export default class EventAdapter {
       x: e.offsetX,
       y: e.offsetY,
     })
-    const evt = e.deltaY < 0 ? 'zoomin' : 'zoomout'
+    const evt = e.deltaY < 0 ? Protocol.INPUT_EV_ZOOM_IN : Protocol.INPUT_EV_ZOOM_OUT
     this.addEvent([evt, pos.x, pos.y])
   }
 }
@@ -363,6 +364,7 @@ async function main() {
   let REPLAY_PAUSED = false
   let lastRealTime = null
   let lastGameTime = null
+  let GAME_START_TS = null
 
   if (MODE === 'play') {
     const game = await Communication.connect(gameId, CLIENT_ID)
@@ -374,7 +376,8 @@ async function main() {
     Game.newGame(game)
     GAME_LOG = log
     lastRealTime = Util.timestamp()
-    lastGameTime = GAME_LOG[0][GAME_LOG[0].length - 1]
+    GAME_START_TS = GAME_LOG[0][GAME_LOG[0].length - 1]
+    lastGameTime = GAME_START_TS
     TIME = () => lastGameTime
   } else {
     throw '[ 2020-12-22 MODE invalid, must be play|replay ]'
@@ -422,22 +425,22 @@ async function main() {
   const evts = new EventAdapter(canvas, viewport)
   if (MODE === 'play') {
     bgColorPickerEl.value = playerBgColor()
-    evts.addEvent(['bg_color', bgColorPickerEl.value])
+    evts.addEvent([Protocol.INPUT_EV_BG_COLOR, bgColorPickerEl.value])
     bgColorPickerEl.addEventListener('change', () => {
       localStorage.setItem('bg_color', bgColorPickerEl.value)
-      evts.addEvent(['bg_color', bgColorPickerEl.value])
+      evts.addEvent([Protocol.INPUT_EV_BG_COLOR, bgColorPickerEl.value])
     })
     playerColorPickerEl.value = playerColor()
-    evts.addEvent(['player_color', playerColorPickerEl.value])
+    evts.addEvent([Protocol.INPUT_EV_PLAYER_COLOR, playerColorPickerEl.value])
     playerColorPickerEl.addEventListener('change', () => {
       localStorage.setItem('player_color', playerColorPickerEl.value)
-      evts.addEvent(['player_color', playerColorPickerEl.value])
+      evts.addEvent([Protocol.INPUT_EV_PLAYER_COLOR, playerColorPickerEl.value])
     })
     nameChangeEl.value = playerName()
-    evts.addEvent(['player_name', nameChangeEl.value])
+    evts.addEvent([Protocol.INPUT_EV_PLAYER_NAME, nameChangeEl.value])
     nameChangeEl.addEventListener('change', () => {
       localStorage.setItem('player_name', nameChangeEl.value)
-      evts.addEvent(['player_name', nameChangeEl.value])
+      evts.addEvent([Protocol.INPUT_EV_PLAYER_NAME, nameChangeEl.value])
     })
   } else if (MODE === 'replay') {
     let setSpeedStatus = () => {
@@ -514,16 +517,23 @@ async function main() {
         }
 
         let logEntry = GAME_LOG[nextIdx]
-        let nextTs = logEntry[logEntry.length - 1]
+        let nextTs = GAME_START_TS + logEntry[logEntry.length - 1]
         if (nextTs > maxGameTs) {
           break
         }
 
-        if (logEntry[0] === 'addPlayer') {
-          Game.addPlayer(gameId, ...logEntry.slice(1))
+        let entryWithTs = logEntry.slice()
+        entryWithTs[entryWithTs.length - 1] = nextTs
+        if (entryWithTs[0] === Protocol.LOG_ADD_PLAYER) {
+          Game.addPlayer(gameId, ...entryWithTs.slice(1))
           RERENDER = true
-        } else if (logEntry[0] === 'handleInput') {
-          Game.handleInput(gameId, ...logEntry.slice(1))
+        } else if (entryWithTs[0] === Protocol.LOG_UPDATE_PLAYER) {
+          let playerId = Game.getPlayerIdByIndex(gameId, entryWithTs[1])
+          Game.addPlayer(gameId, playerId, ...entryWithTs.slice(2))
+          RERENDER = true
+        } else if (entryWithTs[0] === Protocol.LOG_HANDLE_INPUT) {
+          let playerId = Game.getPlayerIdByIndex(gameId, entryWithTs[1])
+          Game.handleInput(gameId, playerId, ...entryWithTs.slice(2))
           RERENDER = true
         }
         GAME_LOG_IDX = nextIdx
@@ -540,7 +550,7 @@ async function main() {
         // LOCAL ONLY CHANGES
         // -------------------------------------------------------------
         const type = evt[0]
-        if (type === 'move') {
+        if (type === Protocol.INPUT_EV_MOUSE_MOVE) {
           if (_last_mouse_down && !Game.getFirstOwnedTile(gameId, CLIENT_ID)) {
             // move the cam
             const pos = { x: evt[1], y: evt[2] }
@@ -552,18 +562,18 @@ async function main() {
 
             _last_mouse_down = mouse
           }
-        } else if (type === 'down') {
+        } else if (type === Protocol.INPUT_EV_MOUSE_DOWN) {
           const pos = { x: evt[1], y: evt[2] }
           _last_mouse_down = viewport.worldToViewport(pos)
-        } else if (type === 'up') {
+        } else if (type === Protocol.INPUT_EV_MOUSE_UP) {
           _last_mouse_down = null
-        } else if (type === 'zoomin') {
+        } else if (type === Protocol.INPUT_EV_ZOOM_IN) {
           if (viewport.zoomIn()) {
             const pos = { x: evt[1], y: evt[2] }
             RERENDER = true
             Game.changePlayer(gameId, CLIENT_ID, pos)
           }
-        } else if (type === 'zoomout') {
+        } else if (type === Protocol.INPUT_EV_ZOOM_OUT) {
           if (viewport.zoomOut()) {
             const pos = { x: evt[1], y: evt[2] }
             RERENDER = true
@@ -583,7 +593,7 @@ async function main() {
         // LOCAL ONLY CHANGES
         // -------------------------------------------------------------
         const type = evt[0]
-        if (type === 'move') {
+        if (type === Protocol.INPUT_EV_MOUSE_MOVE) {
           if (_last_mouse_down) {
             // move the cam
             const pos = { x: evt[1], y: evt[2] }
@@ -595,15 +605,15 @@ async function main() {
 
             _last_mouse_down = mouse
           }
-        } else if (type === 'down') {
+        } else if (type === Protocol.INPUT_EV_MOUSE_DOWN) {
           const pos = { x: evt[1], y: evt[2] }
           _last_mouse_down = viewport.worldToViewport(pos)
-        } else if (type === 'up') {
+        } else if (type === Protocol.INPUT_EV_MOUSE_UP) {
           _last_mouse_down = null
-        } else if (type === 'zoomin') {
+        } else if (type === Protocol.INPUT_EV_ZOOM_IN) {
           viewport.zoomIn()
           RERENDER = true
-        } else if (type === 'zoomout') {
+        } else if (type === Protocol.INPUT_EV_ZOOM_OUT) {
           viewport.zoomOut()
           RERENDER = true
         }
diff --git a/scripts/rewrite_logs.js b/scripts/rewrite_logs.js
new file mode 100644
index 0000000..ae1aae8
--- /dev/null
+++ b/scripts/rewrite_logs.js
@@ -0,0 +1,80 @@
+import fs from 'fs'
+import Protocol from '../common/Protocol.js'
+
+const DATA_DIR = '../data'
+
+const filename = (gameId) => `${DATA_DIR}/log_${gameId}.log`
+
+const rewrite = (gameId) => {
+  const file = filename(gameId)
+  console.log(file)
+  if (!fs.existsSync(file)) {
+    return []
+  }
+  let playerIds = [];
+  let startTs = null
+  const lines = fs.readFileSync(file, 'utf-8').split("\n")
+  const linesNew = lines.filter(line => !!line).map((line) => {
+    const json = JSON.parse(line)
+    const m = {
+      createGame: Protocol.LOG_HEADER,
+      addPlayer: Protocol.LOG_ADD_PLAYER,
+      handleInput: Protocol.LOG_HANDLE_INPUT,
+    }
+    const action = json[0]
+    if (action in m) {
+      json[0] = m[action]
+      if (json[0] === Protocol.LOG_HANDLE_INPUT) {
+        const inputm = {
+          down: Protocol.INPUT_EV_MOUSE_DOWN,
+          up: Protocol.INPUT_EV_MOUSE_UP,
+          move: Protocol.INPUT_EV_MOUSE_MOVE,
+          zoomin: Protocol.INPUT_EV_ZOOM_IN,
+          zoomout: Protocol.INPUT_EV_ZOOM_OUT,
+          bg_color: Protocol.INPUT_EV_BG_COLOR,
+          player_color: Protocol.INPUT_EV_PLAYER_COLOR,
+          player_name: Protocol.INPUT_EV_PLAYER_NAME,
+        }
+        const inputa = json[2][0]
+        if (inputa in inputm) {
+          json[2][0] = inputm[inputa]
+        } else {
+          throw '[ invalid input log line: "' + line + '" ]'
+        }
+      }
+    } else {
+      throw '[ invalid general log line: "' + line + '" ]'
+    }
+
+    if (json[0] === Protocol.LOG_ADD_PLAYER) {
+      if (playerIds.indexOf(json[1]) === -1) {
+        playerIds.push(json[1])
+      } else {
+        json[0] = Protocol.LOG_UPDATE_PLAYER
+        json[1] = playerIds.indexOf(json[1])
+      }
+    }
+
+    if (json[0] === Protocol.LOG_HANDLE_INPUT) {
+      json[1] = playerIds.indexOf(json[1])
+      if (json[1] === -1) {
+        throw '[ invalid player ... "' + line + '" ]'
+      }
+    }
+
+    if (json[0] === Protocol.LOG_HEADER) {
+      startTs = json[json.length - 1]
+      json[4] = json[3]
+      json[3] = json[2]
+      json[2] = json[1]
+      json[1] = 1
+    } else {
+      json[json.length - 1] = json[json.length - 1] - startTs
+    }
+    return JSON.stringify(json)
+  })
+
+  fs.writeFileSync(file, linesNew.join("\n") + "\n")
+}
+
+rewrite(process.argv[2])
diff --git a/server/Game.js b/server/Game.js
index cb72ba5..11a4bb0 100644
--- a/server/Game.js
+++ b/server/Game.js
@@ -4,6 +4,7 @@ import Util from './../common/Util.js'
 import { Rng } from '../common/Rng.js'
 import GameLog from './GameLog.js'
 import { createPuzzle } from './Puzzle.js'
+import Protocol from '../common/Protocol.js'
 
 const DATA_DIR = './../data'
 
@@ -28,6 +29,9 @@ function loadAllGames() {
       let unfinished = game.puzzle.tiles.map(Util.decodeTile).find(t => t.owner !== -1)
       game.puzzle.data.finished = unfinished ? 0 : Util.timestamp()
     }
+    if (!Array.isArray(game.players)) {
+      game.players = Object.values(game.players)
+    }
     GameCommon.newGame({
       id: game.id,
       rng: {
@@ -53,14 +57,14 @@ async function createGameObject(gameId, targetTiles, image, ts) {
       obj: rng,
     },
     await createPuzzle(rng, targetTiles, image, ts),
-    {},
+    [],
     [],
     {}
   )
 }
 async function createGame(gameId, targetTiles, image, ts) {
   GameLog.create(gameId)
-  GameLog.log(gameId, 'createGame', targetTiles, image, ts)
+  GameLog.log(gameId, Protocol.LOG_HEADER, 1, targetTiles, image, ts)
 
   const seed = Util.hash(gameId + ' ' + ts)
   const rng = new Rng(seed)
@@ -71,7 +75,7 @@ async function createGame(gameId, targetTiles, image, ts) {
       obj: rng,
     },
     puzzle: await createPuzzle(rng, targetTiles, image, ts),
-    players: {},
+    players: [],
     sockets: [],
     evtInfos: {},
   })
@@ -80,7 +84,15 @@ async function createGame(gameId, targetTiles, image, ts) {
 }
 
 function addPlayer(gameId, playerId, ts) {
-  GameLog.log(gameId, 'addPlayer', playerId, ts)
+  const idx = GameCommon.getPlayerIndexById(gameId, playerId)
+  if (idx === -1) {
+    const diff = ts - GameCommon.getStartTs(gameId)
+    GameLog.log(gameId, Protocol.LOG_ADD_PLAYER, playerId, diff)
+  } else {
+    const diff = ts - GameCommon.getStartTs(gameId)
+    GameLog.log(gameId, Protocol.LOG_UPDATE_PLAYER, idx, diff)
+  }
+
   GameCommon.addPlayer(gameId, playerId, ts)
   changedGames[gameId] = true
 }
@@ -91,7 +103,9 @@ function addSocket(gameId, socket) {
 }
 
 function handleInput(gameId, playerId, input, ts) {
-  GameLog.log(gameId, 'handleInput', playerId, input, ts)
+  const idx = GameCommon.getPlayerIndexById(gameId, playerId)
+  const diff = ts - GameCommon.getStartTs(gameId)
+  GameLog.log(gameId, Protocol.LOG_HANDLE_INPUT, idx, input, diff)
 
   const ret = GameCommon.handleInput(gameId, playerId, input, ts)
   changedGames[gameId] = true
diff --git a/server/index.js b/server/index.js
index 4d30143..1ea29cb 100644
--- a/server/index.js
+++ b/server/index.js
@@ -140,9 +140,9 @@ wss.on('message', async ({socket, data}) => {
         const log = GameLog.get(gameId)
         let game = await Game.createGameObject(
           gameId,
-          log[0][1],
           log[0][2],
-          log[0][3]
+          log[0][3],
+          log[0][4]
         )
         notify(
           [Protocol.EV_SERVER_INIT_REPLAY, {