diff --git a/OpenRA.Game/Game.cs b/OpenRA.Game/Game.cs index b12abb8c3336..81326a408b0e 100644 --- a/OpenRA.Game/Game.cs +++ b/OpenRA.Game/Game.cs @@ -33,9 +33,11 @@ public static class Game { public const int DefaultNetTickScale = 3; // 120ms net tick for 40ms timestep public const int NewNetcodeNetTickScale = 1; // Net tick every world frame + public const int MaxNetworkSimLag = 50; + public const int MaxSimLagBeforeFrameDrops = 50; public const int Timestep = 40; public const int TimestepJankThreshold = 250; // Don't catch up for delays larger than 250ms - public const double NetCatchupFactor = 0.1; + public const double NetCatchupFactor = 0.2; public static InstalledMods Mods { get; private set; } public static ExternalMods ExternalMods { get; private set; } @@ -595,17 +597,7 @@ static void InnerLogicTick(OrderManager orderManager) Cursor.Tick(); } - int worldTimestep; - if (world == null) - worldTimestep = Timestep; - else if (world.IsLoadingGameSave) - worldTimestep = 1; - else if (orderManager.IsStalling) - worldTimestep = 1; - else if (orderManager.CatchUpFrames > 0) - worldTimestep = (int)Math.Floor(world.Timestep / (1.0 + NetCatchupFactor * orderManager.CatchUpFrames)); // Smooth catchup - else - worldTimestep = world.Timestep; + var worldTimestep = orderManager.SuggestedTimestep; var worldTickDelta = tick - orderManager.LastTickTime; if (worldTimestep == 0 || worldTickDelta < worldTimestep) @@ -613,12 +605,22 @@ static void InnerLogicTick(OrderManager orderManager) using (new PerfSample("tick_time")) { - // Tick the world to advance the world time to match real time: - // If dt < TickJankThreshold then we should try and catch up by repeatedly ticking - // If dt >= TickJankThreshold then we should accept the jank and progress at the normal rate - // dt is rounded down to an integer tick count in order to preserve fractional tick components. - var integralTickTimestep = (worldTickDelta / worldTimestep) * worldTimestep; - orderManager.LastTickTime += integralTickTimestep >= TimestepJankThreshold ? integralTickTimestep : worldTimestep; + if (world == null || !orderManager.ShouldUseCatchUp || !orderManager.LobbyInfo.GlobalSettings.UseNewNetcode) + { + // Tick the world to advance the world time to match real time: + // If dt < TickJankThreshold then we should try and catch up by repeatedly ticking + // If dt >= TickJankThreshold then we should accept the jank and progress at the normal rate + // dt is rounded down to an integer tick count in order to preserve fractional tick components. + var integralTickTimestep = (worldTickDelta / worldTimestep) * worldTimestep; + orderManager.LastTickTime += integralTickTimestep >= TimestepJankThreshold ? integralTickTimestep : worldTimestep; + } + // else + // { + // // Console.WriteLine("stalled {0}", worldTimestep); + // orderManager.LastTickTime = tick; + // } + + orderManager.RealTickTime = tick; Sound.Tick(); @@ -799,23 +801,16 @@ static void Loop() { // Ideal time between logic updates. Timestep = 0 means the game is paused // but we still call LogicTick() because it handles pausing internally. - var logicInterval = worldRenderer != null && worldRenderer.World.Timestep != 0 ? worldRenderer.World.Timestep : Timestep; + // var logicInterval = worldRenderer != null && worldRenderer.World.Timestep != 0 ? worldRenderer.World.Timestep : OrderManager.SuggestedTimestep; + var logicInterval = 1; // Ideal time between screen updates var maxFramerate = Settings.Graphics.CapFramerate ? Settings.Graphics.MaxFramerate.Clamp(1, 1000) : 1000; var renderInterval = 1000 / maxFramerate; - if (OrderManager.IsStalling) - logicInterval = 1; - - // TODO: limit rendering if we are taking too long to catch up - else if (OrderManager.CatchUpFrames > 0) - logicInterval = (int)Math.Floor(logicInterval / (1.0 + NetCatchupFactor * OrderManager.CatchUpFrames)); - - // Tick as fast as possible while restoring game saves, capping rendering at 5 FPS + // Whilst restoring game saves, cap rendering at 5 FPS if (OrderManager.World != null && OrderManager.World.IsLoadingGameSave) { - logicInterval = 1; renderInterval = 200; } @@ -827,18 +822,20 @@ static void Loop() // When's the next update (logic or render) var nextUpdate = Math.Min(nextLogic, nextRender); + if (now >= nextUpdate) { var forceRender = renderBeforeNextTick || now >= forcedNextRender; if (now >= nextLogic && !renderBeforeNextTick) { - nextLogic += logicInterval; + // nextLogic += logicInterval; + nextLogic = now + logicInterval; LogicTick(); // Force at least one render per tick during regular gameplay - if (!OrderManager.IsStalling && !(OrderManager.CatchUpFrames > 2) && OrderManager.World != null && !OrderManager.World.IsLoadingGameSave && !OrderManager.World.IsReplay) + if (!OrderManager.IsStalling && OrderManager.SimLag < MaxSimLagBeforeFrameDrops && OrderManager.World != null && !OrderManager.World.IsLoadingGameSave && !OrderManager.World.IsReplay) renderBeforeNextTick = true; } diff --git a/OpenRA.Game/Network/Connection.cs b/OpenRA.Game/Network/Connection.cs index b66e0629af64..5fcb117d4998 100644 --- a/OpenRA.Game/Network/Connection.cs +++ b/OpenRA.Game/Network/Connection.cs @@ -38,7 +38,7 @@ public interface IConnection : IDisposable void Send(int frame, IEnumerable orders); void SendImmediate(IEnumerable orders); void SendSync(int frame, byte[] syncData); - void Receive(Action packetFn); + void Receive(Action packetFn); } public class ConnectionTarget @@ -96,6 +96,7 @@ protected struct ReceivedPacket { public int FromClient; public byte[] Data; + public int Timestep; } readonly ConcurrentBag receivedPackets = new ConcurrentBag(); @@ -162,7 +163,7 @@ protected void AddPacket(ReceivedPacket packet) receivedPackets.Add(packet); } - public virtual void Receive(Action packetFn) + public virtual void Receive(Action packetFn) { var packets = new List(receivedPackets.Count); @@ -173,7 +174,7 @@ public virtual void Receive(Action packetFn) foreach (var p in packets) { - packetFn(p.FromClient, p.Data); + packetFn(p.FromClient, p.Data, p.Timestep); Recorder?.Receive(p.FromClient, p.Data); } } @@ -313,7 +314,7 @@ void NetworkConnectionReceive() var client = reader.ReadInt32(); var buf = reader.ReadBytes(len); - if (UseNewNetcode && client == LocalClientId && len == 7 && buf[4] == (byte)OrderType.Ack) + if (UseNewNetcode && client == LocalClientId && len == 9 && buf[4] == (byte)OrderType.Ack) { Ack(buf); } @@ -336,34 +337,18 @@ void NetworkConnectionReceive() void Ack(byte[] buf) { - int frameReceived; - short framesToAck; - using (var reader = new BinaryReader(new MemoryStream(buf))) - { - frameReceived = reader.ReadInt32(); - reader.ReadByte(); - framesToAck = reader.ReadInt16(); - } + var reader = new BinaryReader(new MemoryStream(buf)); + var frameReceived = reader.ReadInt32(); + reader.ReadByte(); + var framesToAck = reader.ReadInt16(); + var timestep = reader.ReadInt16(); var ms = new MemoryStream(4 + awaitingAckPackets.Take(framesToAck).Sum(i => i.Length)); ms.WriteArray(BitConverter.GetBytes(frameReceived)); for (var i = 0; i < framesToAck; i++) { - byte[] queuedPacket = default; - if (awaitingAckPackets.Count > 0 && !awaitingAckPackets.TryDequeue(out queuedPacket)) - { - // The dequeuing failed because of concurrency, so we retry - for (var c = 0; c < 5; c++) - { - if (awaitingAckPackets.TryDequeue(out queuedPacket)) - { - break; - } - } - } - - if (queuedPacket == default) + if (!awaitingAckPackets.TryDequeue(out var queuedPacket)) { throw new InvalidOperationException("Received acks for unknown frames"); } @@ -371,7 +356,7 @@ void Ack(byte[] buf) ms.WriteArray(queuedPacket); } - AddPacket(new ReceivedPacket { FromClient = LocalClientId, Data = ms.GetBuffer() }); + AddPacket(new ReceivedPacket { FromClient = LocalClientId, Data = ms.GetBuffer(), Timestep = timestep }); } public override int LocalClientId { get { return clientId; } } diff --git a/OpenRA.Game/Network/FrameData.cs b/OpenRA.Game/Network/FrameData.cs index 5d9fdc162440..70e103f0b3c7 100644 --- a/OpenRA.Game/Network/FrameData.cs +++ b/OpenRA.Game/Network/FrameData.cs @@ -29,6 +29,7 @@ public override string ToString() readonly HashSet quitClients = new HashSet(); readonly Dictionary> framePackets = new Dictionary>(); + readonly Queue timestepData = new Queue(); public IEnumerable ClientsPlayingInFrame() { @@ -46,7 +47,7 @@ public void ClientQuit(int clientId) quitClients.Add(clientId); } - public void AddFrameOrders(int clientId, byte[] orders) + public void AddFrameOrders(int clientId, byte[] orders, int timestep) { // HACK: Due to design we can actually receive client orders before the game start order // has been acted on, since immediate orders are buffered, so not all clients will have @@ -56,6 +57,9 @@ public void AddFrameOrders(int clientId, byte[] orders) var frameData = framePackets[clientId]; frameData.Enqueue(orders); + + if (timestep != 0) + timestepData.Enqueue(timestep); } public bool IsReadyForFrame() @@ -80,5 +84,20 @@ public int BufferSizeForClient(int client) { return framePackets[client].Count; } + + public bool TryPeekTimestep(out int timestep) + { + return timestepData.TryPeek(out timestep); + } + + public void AdvanceFrame() + { + timestepData.TryDequeue(out _); + } + + public int BufferTimeRemaining() + { + return timestepData.Sum(); + } } } diff --git a/OpenRA.Game/Network/OrderManager.cs b/OpenRA.Game/Network/OrderManager.cs index b1a19c9ff6a9..10ab1b319176 100644 --- a/OpenRA.Game/Network/OrderManager.cs +++ b/OpenRA.Game/Network/OrderManager.cs @@ -19,6 +19,11 @@ namespace OpenRA.Network { public sealed class OrderManager : IDisposable { + const double CatchupFactor = 0.1; + const double CatchUpLimit = 1.5; + const int SimLagThreshold = 500; + const int SimLagLimit = 1000; + static readonly IEnumerable NoClients = new Session.Client[] { }; readonly SyncReport syncReport; @@ -46,10 +51,30 @@ public sealed class OrderManager : IDisposable public bool ShouldUseCatchUp; public int OrderLatency; // Set during lobby by a "SyncInfo" packet, see UnitOrders public int NextOrderFrame; - public int CatchUpFrames { get; private set; } - public bool IsStalling { get; private set; } + public int CatchUpAmount { get; private set; } + public int SuggestedTimestep + { + get + { + if (World == null) + return Game.Timestep; + + if (IsStalling || World.IsLoadingGameSave) + return 1; + + return TargetTimestep; + } + } + // int suggestedCatchupTimestep; + public volatile bool IsStalling; public long LastTickTime = Game.RunTime; + public long RealTickTime = Game.RunTime; + public long OldRealTickTime = Game.RunTime; + public int TargetTimestep; + public int ActualTimestep; + public double AverageSimLag; + public int SimLag = 0; public bool GameStarted { get { return NetFrameNumber != 0; } } public IConnection Connection { get; private set; } @@ -93,6 +118,11 @@ public void StartGame() NetFrameNumber = 1; NextOrderFrame = 1; + // suggestedCatchupTimestep = World.Timestep; + SimLag = 0; + TargetTimestep = World.Timestep; + ActualTimestep = World.Timestep; + if (LobbyInfo.GlobalSettings.UseNewNetcode) localImmediateOrders.Add(Order.FromTargetString("Loaded", "", true)); else @@ -144,7 +174,7 @@ void SendImmediateOrders() void ReceiveAllOrdersAndCheckSync() { Connection.Receive( - (clientId, packet) => + (clientId, packet, timestep) => { var frame = BitConverter.ToInt32(packet, 0); if (packet.Length == 5 && packet[4] == (byte)OrderType.Disconnect) @@ -154,7 +184,7 @@ void ReceiveAllOrdersAndCheckSync() else if (frame == 0) immediatePackets.Add((clientId, packet)); else - frameData.AddFrameOrders(clientId, packet); + frameData.AddFrameOrders(clientId, packet, timestep); }); } @@ -195,18 +225,42 @@ void CheckSync(byte[] packet) void CompensateForLatency() { - // NOTE: subtract 1 because we are only interested in *excess* frames - var catchUpNetFrames = frameData.BufferSizeForClient(Connection.LocalClientId) - 1; - if (catchUpNetFrames < 0) - catchUpNetFrames = 0; + if (!LobbyInfo.GlobalSettings.UseNewNetcode || !ShouldUseCatchUp) + return; + + if (LocalFrameNumber == 0 || IsStalling) + { + LastTickTime = RealTickTime; + OldRealTickTime = RealTickTime; + TargetTimestep = ActualTimestep; + SimLag = (SimLag - 1).Clamp(-World.Timestep, SimLagLimit); // We are happy to stall after a lag spike + return; + } + + var bufferRemaining = frameData.BufferSizeForClient(LocalClient.Index) * World.Timestep; - CatchUpFrames = ShouldUseCatchUp ? catchUpNetFrames : 0; + var realTimestep = (int)(RealTickTime - OldRealTickTime); - if (LastSlowDownRequestTick + 5 < NetFrameNumber && (catchUpNetFrames > 5)) + SimLag = (SimLag + realTimestep - TargetTimestep).Clamp(-World.Timestep, SimLagLimit); + + var catchup = (int)Math.Ceiling(bufferRemaining * CatchupFactor).Clamp(0, World.Timestep / CatchUpLimit); + + var simLagDelta = realTimestep - World.Timestep + catchup; + + AverageSimLag = AverageSimLag * 0.95 + simLagDelta * 0.05; + var slowDownRequired = (int)Math.Ceiling(AverageSimLag); + + if (slowDownRequired > 0 && SimLag > SimLagThreshold && NetFrameNumber > LastSlowDownRequestTick + 5) { - localImmediateOrders.Add(Order.FromTargetString("SlowDown", catchUpNetFrames.ToString(), true)); + localImmediateOrders.Add(Order.FromTargetString("SlowDown", slowDownRequired.ToString(), true)); LastSlowDownRequestTick = NetFrameNumber; } + + TargetTimestep = (ActualTimestep - catchup).Clamp(1, 1000); + + LastTickTime = RealTickTime - SimLag; + + OldRealTickTime = RealTickTime; } IEnumerable GetClientsNotReadyForNextFrame @@ -263,6 +317,11 @@ void ProcessOrders() syncReport.UpdateSyncReport(orders); } + if (frameData.TryPeekTimestep(out var timestep)) + ActualTimestep = timestep; + + frameData.AdvanceFrame(); + ++NetFrameNumber; } @@ -291,10 +350,6 @@ public bool TryTick() SendOrders(); } - // Sets catchup frames and asks server to slow down if they are too high - if (LobbyInfo.GlobalSettings.UseNewNetcode) - CompensateForLatency(); - SendImmediateOrders(); ReceiveAllOrdersAndCheckSync(); @@ -308,13 +363,15 @@ public bool TryTick() willTick = frameData.IsReadyForFrame(); if (willTick) ProcessOrders(); - - IsStalling = !willTick; } if (willTick) LocalFrameNumber++; + IsStalling = !willTick; + + CompensateForLatency(); + return willTick; } diff --git a/OpenRA.Game/Network/ReplayConnection.cs b/OpenRA.Game/Network/ReplayConnection.cs index 0ece4ad15087..570760878ba3 100644 --- a/OpenRA.Game/Network/ReplayConnection.cs +++ b/OpenRA.Game/Network/ReplayConnection.cs @@ -144,21 +144,21 @@ public void SendSync(int frame, byte[] syncData) ordersFrame = frame + 1; } - public void Receive(Action packetFn) + public void Receive(Action packetFn) { while (sync.Count != 0) - packetFn(LocalClientId, sync.Dequeue()); + packetFn(LocalClientId, sync.Dequeue(), 0); while (chunks.Count != 0 && chunks.Peek().Frame <= ordersFrame) foreach (var o in chunks.Dequeue().Packets) - packetFn(o.ClientId, o.Packet); + packetFn(o.ClientId, o.Packet, 0); // Stream ended, disconnect everyone if (chunks.Count == 0) { var disconnectPacket = new byte[] { 0, 0, 0, 0, (byte)OrderType.Disconnect }; foreach (var client in LobbyInfo.Clients) - packetFn(client.Index, disconnectPacket); + packetFn(client.Index, disconnectPacket, 0); } } diff --git a/OpenRA.Game/Server/OrderBuffer.cs b/OpenRA.Game/Server/OrderBuffer.cs index 98748cca57ac..0c9324cf3ace 100644 --- a/OpenRA.Game/Server/OrderBuffer.cs +++ b/OpenRA.Game/Server/OrderBuffer.cs @@ -56,7 +56,7 @@ public void DropClient(int client) // From, To, EnumerableData // Then clears the buffer // TODO allow server to optionally store buffered frames and enable client joins and re-connects - public void DispatchOrders(IFrameOrderDispatcher dispatcher) + public void DispatchOrders(IFrameOrderDispatcher dispatcher, int timestep) { foreach (var fromPair in clientOrdersBuffer) { @@ -64,7 +64,7 @@ public void DispatchOrders(IFrameOrderDispatcher dispatcher) var orders = fromPair.Value; // Ack the frames sent to be applied on this frame - dispatcher.DispatchBufferedOrderAcks(fromClient, orders.Count); + dispatcher.DispatchBufferedOrderAcks(fromClient, orders.Count, timestep); // Send each client's order buffer (because they were queued, order is preserved) dispatcher.DispatchBufferedOrdersToOtherClients(fromClient, orders); @@ -77,6 +77,6 @@ public void DispatchOrders(IFrameOrderDispatcher dispatcher) public interface IFrameOrderDispatcher { void DispatchBufferedOrdersToOtherClients(int fromClient, List allData); - void DispatchBufferedOrderAcks(int forClient, int ackCount); + void DispatchBufferedOrderAcks(int forClient, int ackCount, int timestep); } } diff --git a/OpenRA.Game/Server/Server.cs b/OpenRA.Game/Server/Server.cs index 43bb2d95bc37..3e010591c345 100644 --- a/OpenRA.Game/Server/Server.cs +++ b/OpenRA.Game/Server/Server.cs @@ -873,15 +873,16 @@ public void DispatchBufferedOrdersToOtherClients(int fromClient, List al DispatchOrdersToOtherClients(conn, serverGame.CurrentNetFrame, ms.ToArray(), true); } - public void DispatchBufferedOrderAcks(int forClient, int acks) + public void DispatchBufferedOrderAcks(int forClient, int acks, int timestep) { if (acks > 0xFFFF) throw new InvalidOperationException("Acks too great"); - var ms = new MemoryStream(3); + var ms = new MemoryStream(5); var writer = new BinaryWriter(ms); writer.Write((byte)OrderType.Ack); writer.Write((short)acks); + writer.Write((short)timestep); var conn = Conns.FirstOrDefault(c => c.PlayerIndex == forClient); diff --git a/OpenRA.Game/Server/ServerGame.cs b/OpenRA.Game/Server/ServerGame.cs index 8e932bb1b1d7..a2cc1a128260 100644 --- a/OpenRA.Game/Server/ServerGame.cs +++ b/OpenRA.Game/Server/ServerGame.cs @@ -17,7 +17,7 @@ public class ServerGame { const int JankThreshold = 250; - Stopwatch gameTimer; + readonly Stopwatch gameTimer; public long RunTime { get { return gameTimer.ElapsedMilliseconds; } @@ -30,7 +30,7 @@ public long RunTime int slowdownHold; int slowdownAmount; - public int AdjustedTimestep { get { return NetTimestep + slowdownAmount; } } + public int AdjustedTimestep { get { return (NetTimestep + slowdownAmount).Clamp(1, 1000); } } public int MillisToNextNetFrame { @@ -53,13 +53,15 @@ public bool TryTick(IFrameOrderDispatcher dispatcher) if (now < NextFrameTick) return false; - OrderBuffer.DispatchOrders(dispatcher); + var timestep = AdjustedTimestep; + + OrderBuffer.DispatchOrders(dispatcher, timestep); CurrentNetFrame++; if (now - NextFrameTick > JankThreshold) - NextFrameTick = now + AdjustedTimestep; + NextFrameTick = now + timestep; else - NextFrameTick += AdjustedTimestep; + NextFrameTick += timestep; if (slowdownHold > 0) slowdownHold--; @@ -75,7 +77,7 @@ public void SlowDown(int amount) if (slowdownAmount < amount) { slowdownAmount = amount; - slowdownHold = amount; + slowdownHold = 5; } } }