fix arena evaluation bugs

Author: huangshiyu13

examples/arena/run_arena.py

@@ -52,3 +52,4 @@ def run_arena(
 
 if __name__ == "__main__":
     run_arena(render=False, parallel=True, seed=0, total_games=100, max_game_onetime=10)
+    # run_arena(render=True, parallel=True, seed=1, total_games=10, max_game_onetime=2)

examples/selfplay/human_vs_agent.py

@@ -27,7 +27,7 @@ def get_human_env(env_num):
     env = make(
         "tictactoe_v3",
         env_num=env_num,
-        asynchronous=False,
+        asynchronous=True,
         opponent_wrappers=[TictactoeRender, HumanOpponentWrapper],
         env_wrappers=[FlattenObservation],
         auto_reset=False,

examples/selfplay/tictactoe_utils/tictactoe_render.py

@@ -48,16 +48,6 @@ def step(self, action: ActionType) -> None:
         self.last_action = action[0]
         return result
 
-    def observe(self, agent: str) -> Optional[ObsType]:
-        obs = super().observe(agent)
-        if self.last_action is not None:
-            if self.render_mode == "game":
-                self.game.make_move(self.last_action // 3, self.last_action % 3)
-                pygame.display.update()
-            self.last_action = None
-            time.sleep(0.3)
-        return obs
-
     def close(self):
         super().close()
         self.game.close()
@@ -69,3 +59,12 @@ def get_human_action(self, agent, observation, termination, truncation, info):
         return self.game.get_human_action(
             agent, observation, termination, truncation, info
         )
+
+    def last(self, observe: bool = True):
+        if self.last_action is not None:
+            if self.render_mode == "game":
+                self.game.make_move(self.last_action // 3, self.last_action % 3)
+                pygame.display.update()
+            self.last_action = None
+            time.sleep(0.3)
+        return self.env.last(observe)

openrl/arena/games/two_player_game.py

@@ -54,6 +54,7 @@ def _run(self, env_fn: Callable, agents: List[BaseAgent]):
             info = {}
             for player_name in env.agent_iter():
                 observation, reward, termination, truncation, info = env.last()
+
                 if termination:
                     break
                 action = player2agent[player_name].act(

openrl/envs/wrappers/pettingzoo_wrappers.py

@@ -77,18 +77,6 @@ def reset(self, seed: Optional[int] = None, options: Optional[Dict] = None):
         self.total_rewards = defaultdict(float)
         return super().reset(seed, options)
 
-    def step(self, action: ActionType) -> None:
-        super().step(action)
-        winners = None
-        losers = None
-        for agent in self.terminations:
-            if self.terminations[agent]:
-                if winners is None:
-                    winners = self.get_winners()
-                    losers = [player for player in self.agents if player not in winners]
-                self.infos[agent]["winners"] = winners
-                self.infos[agent]["losers"] = losers
-
     def get_winners(self):
         max_reward = max(self.total_rewards.values())
 
@@ -101,11 +89,21 @@ def get_winners(self):
 
     def last(self, observe: bool = True):
         """Returns observation, cumulative reward, terminated, truncated, info for the current agent (specified by self.agent_selection)."""
+
         agent = self.agent_selection
-        # if self._cumulative_rewards[agent]!=0:
-        #     print("agent:",agent,self._cumulative_rewards[agent])
+        # this may be miss the last reward for another agent
         self.total_rewards[agent] += self._cumulative_rewards[agent]
 
+        winners = None
+        losers = None
+        for agent in self.terminations:
+            if self.terminations[agent]:
+                if winners is None:
+                    winners = self.get_winners()
+                    losers = [player for player in self.agents if player not in winners]
+                self.infos[agent]["winners"] = winners
+                self.infos[agent]["losers"] = losers
+
         return super().last(observe)