Only do physics and collision calculations on the clients side, and only for their own entity.

src/Engine/Collision/CollisionSystem.cpp

@@ -17,6 +17,7 @@ void CollisionSystem::UpdateComponent(EntityWrapper& entity, ComponentWrapper& c
     EntityAABB& boxA = *boundingBox;
     bool everHitTheGround = false;
 
+    if (entity == LocalPlayer) {
         auto prevPosIt = m_PrevPositions.find(entity);
         if (prevPosIt != m_PrevPositions.end()) {
             glm::vec3 size = boxA.Size();
@@ -119,3 +120,4 @@ void CollisionSystem::UpdateComponent(EntityWrapper& entity, ComponentWrapper& c
 
         m_PrevPositions[entity] = boxA.Origin();
     }
+}

src/Game/Systems/PlayerMovementSystem.cpp

@@ -18,16 +18,11 @@ PlayerMovementSystem::~PlayerMovementSystem()
 void PlayerMovementSystem::Update(double dt)
 {
     updateMovementControllers(dt);
-    if (IsServer) {
-        for (auto& kv : m_PlayerInputControllers) {
-            updateVelocity(kv.first, dt);
-        }
-    } else {
-        if (LocalPlayer.Valid()) {
+    // Only do physics calculations on client and only for themselves.
+    if (!IsServer && LocalPlayer.Valid()) {
         updateVelocity(LocalPlayer, dt);
     }
 }
-}
 
 void PlayerMovementSystem::updateMovementControllers(double dt)
 {