fix 4 steps

rlcube/cube2.ipynb

@@ -2,7 +2,7 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": 127,
+   "execution_count": 55,
    "id": "dff864f2",
    "metadata": {},
    "outputs": [],
@@ -231,7 +231,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 128,
+   "execution_count": 56,
    "id": "624c83c1",
    "metadata": {},
    "outputs": [],
@@ -243,10 +243,14 @@
     "    def state(self):\n",
     "        return self.env.state\n",
     "    \n",
+    "    def step_count(self):\n",
+    "        return self.env.step_count\n",
+    "    \n",
     "    def reset(self, *args, **kwargs):\n",
-    "        super().reset(*args, **kwargs)\n",
-    "        self.env.step(self.env.action_space.sample())\n",
-    "        self.env.step(self.env.action_space.sample())\n",
+    "        self.env.reset(*args, **kwargs)\n",
+    "        for _ in range(4):\n",
+    "            self.env.step(self.env.action_space.sample())\n",
+    "        self.env.step_count = 0\n",
     "        return self.env._get_obs(), {}\n",
     "\n",
     "    def step(self, action):\n",
@@ -258,7 +262,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 130,
+   "execution_count": null,
    "id": "f8b4d968",
    "metadata": {},
    "outputs": [
@@ -271,333 +275,183 @@
       "Wrapping the env in a DummyVecEnv.\n",
       "----------------------------------\n",
       "| rollout/            |          |\n",
-      "|    ep_len_mean      | 91.4     |\n",
-      "|    ep_rew_mean      | -84.3    |\n",
-      "|    exploration_rate | 0.132    |\n",
+      "|    ep_len_mean      | 94.2     |\n",
+      "|    ep_rew_mean      | -88.2    |\n",
+      "|    exploration_rate | 0.105    |\n",
       "| time/               |          |\n",
       "|    episodes         | 100      |\n",
-      "|    fps              | 4624     |\n",
+      "|    fps              | 4943     |\n",
       "|    time_elapsed     | 1        |\n",
-      "|    total_timesteps  | 9136     |\n",
+      "|    total_timesteps  | 9424     |\n",
       "| train/              |          |\n",
       "|    learning_rate    | 0.0001   |\n",
-      "|    loss             | 0.00031  |\n",
-      "|    n_updates        | 2258     |\n",
+      "|    loss             | 0.0004   |\n",
+      "|    n_updates        | 2330     |\n",
       "----------------------------------\n",
       "----------------------------------\n",
       "| rollout/            |          |\n",
-      "|    ep_len_mean      | 87.9     |\n",
-      "|    ep_rew_mean      | -76.8    |\n",
+      "|    ep_len_mean      | 98.1     |\n",
+      "|    ep_rew_mean      | -96.1    |\n",
       "|    exploration_rate | 0.05     |\n",
       "| time/               |          |\n",
       "|    episodes         | 200      |\n",
-      "|    fps              | 4407     |\n",
+      "|    fps              | 4426     |\n",
       "|    time_elapsed     | 4        |\n",
-      "|    total_timesteps  | 17928    |\n",
+      "|    total_timesteps  | 19236    |\n",
       "| train/              |          |\n",
       "|    learning_rate    | 0.0001   |\n",
-      "|    loss             | 0.00032  |\n",
-      "|    n_updates        | 4456     |\n",
+      "|    loss             | 0.000292 |\n",
+      "|    n_updates        | 4783     |\n",
       "----------------------------------\n",
       "----------------------------------\n",
       "| rollout/            |          |\n",
-      "|    ep_len_mean      | 80.2     |\n",
-      "|    ep_rew_mean      | -61      |\n",
+      "|    ep_len_mean      | 95.2     |\n",
+      "|    ep_rew_mean      | -90.1    |\n",
       "|    exploration_rate | 0.05     |\n",
       "| time/               |          |\n",
       "|    episodes         | 300      |\n",
-      "|    fps              | 4300     |\n",
+      "|    fps              | 4349     |\n",
       "|    time_elapsed     | 6        |\n",
-      "|    total_timesteps  | 25946    |\n",
+      "|    total_timesteps  | 28754    |\n",
       "| train/              |          |\n",
       "|    learning_rate    | 0.0001   |\n",
-      "|    loss             | 0.000486 |\n",
-      "|    n_updates        | 6461     |\n",
+      "|    loss             | 0.000103 |\n",
+      "|    n_updates        | 7163     |\n",
       "----------------------------------\n",
       "----------------------------------\n",
       "| rollout/            |          |\n",
-      "|    ep_len_mean      | 71.3     |\n",
-      "|    ep_rew_mean      | -43      |\n",
+      "|    ep_len_mean      | 88.4     |\n",
+      "|    ep_rew_mean      | -76.3    |\n",
       "|    exploration_rate | 0.05     |\n",
       "| time/               |          |\n",
       "|    episodes         | 400      |\n",
-      "|    fps              | 4189     |\n",
-      "|    time_elapsed     | 7        |\n",
-      "|    total_timesteps  | 33072    |\n",
+      "|    fps              | 4391     |\n",
+      "|    time_elapsed     | 8        |\n",
+      "|    total_timesteps  | 37598    |\n",
       "| train/              |          |\n",
       "|    learning_rate    | 0.0001   |\n",
-      "|    loss             | 0.000479 |\n",
-      "|    n_updates        | 8242     |\n",
+      "|    loss             | 0.000121 |\n",
+      "|    n_updates        | 9374     |\n",
       "----------------------------------\n",
       "----------------------------------\n",
       "| rollout/            |          |\n",
-      "|    ep_len_mean      | 62.8     |\n",
-      "|    ep_rew_mean      | -23.4    |\n",
+      "|    ep_len_mean      | 86.6     |\n",
+      "|    ep_rew_mean      | -72.5    |\n",
       "|    exploration_rate | 0.05     |\n",
       "| time/               |          |\n",
       "|    episodes         | 500      |\n",
-      "|    fps              | 4123     |\n",
-      "|    time_elapsed     | 9        |\n",
-      "|    total_timesteps  | 39348    |\n",
-      "| train/              |          |\n",
-      "|    learning_rate    | 0.0001   |\n",
-      "|    loss             | 0.000449 |\n",
-      "|    n_updates        | 9811     |\n",
-      "----------------------------------\n",
-      "----------------------------------\n",
-      "| rollout/            |          |\n",
-      "|    ep_len_mean      | 54.2     |\n",
-      "|    ep_rew_mean      | -6.69    |\n",
-      "|    exploration_rate | 0.05     |\n",
-      "| time/               |          |\n",
-      "|    episodes         | 600      |\n",
-      "|    fps              | 4072     |\n",
+      "|    fps              | 4417     |\n",
       "|    time_elapsed     | 10       |\n",
-      "|    total_timesteps  | 44764    |\n",
-      "| train/              |          |\n",
-      "|    learning_rate    | 0.0001   |\n",
-      "|    loss             | 0.000499 |\n",
-      "|    n_updates        | 11165    |\n",
-      "----------------------------------\n",
-      "----------------------------------\n",
-      "| rollout/            |          |\n",
-      "|    ep_len_mean      | 37.5     |\n",
-      "|    ep_rew_mean      | 27.1     |\n",
-      "|    exploration_rate | 0.05     |\n",
-      "| time/               |          |\n",
-      "|    episodes         | 700      |\n",
-      "|    fps              | 4063     |\n",
-      "|    time_elapsed     | 11       |\n",
-      "|    total_timesteps  | 48514    |\n",
+      "|    total_timesteps  | 46260    |\n",
       "| train/              |          |\n",
       "|    learning_rate    | 0.0001   |\n",
-      "|    loss             | 0.000346 |\n",
-      "|    n_updates        | 12103    |\n",
+      "|    loss             | 0.000169 |\n",
+      "|    n_updates        | 11539    |\n",
       "----------------------------------\n",
       "----------------------------------\n",
       "| rollout/            |          |\n",
-      "|    ep_len_mean      | 38.3     |\n",
-      "|    ep_rew_mean      | 26.3     |\n",
+      "|    ep_len_mean      | 82.6     |\n",
+      "|    ep_rew_mean      | -64.4    |\n",
       "|    exploration_rate | 0.05     |\n",
       "| time/               |          |\n",
-      "|    episodes         | 800      |\n",
-      "|    fps              | 4067     |\n",
+      "|    episodes         | 600      |\n",
+      "|    fps              | 4436     |\n",
       "|    time_elapsed     | 12       |\n",
-      "|    total_timesteps  | 52346    |\n",
+      "|    total_timesteps  | 54520    |\n",
       "| train/              |          |\n",
       "|    learning_rate    | 0.0001   |\n",
-      "|    loss             | 0.000947 |\n",
-      "|    n_updates        | 13061    |\n",
+      "|    loss             | 9.72e-05 |\n",
+      "|    n_updates        | 13604    |\n",
       "----------------------------------\n",
       "----------------------------------\n",
       "| rollout/            |          |\n",
-      "|    ep_len_mean      | 37.5     |\n",
-      "|    ep_rew_mean      | 28.2     |\n",
+      "|    ep_len_mean      | 79.4     |\n",
+      "|    ep_rew_mean      | -57.2    |\n",
       "|    exploration_rate | 0.05     |\n",
       "| time/               |          |\n",
-      "|    episodes         | 900      |\n",
-      "|    fps              | 4076     |\n",
-      "|    time_elapsed     | 13       |\n",
-      "|    total_timesteps  | 56094    |\n",
-      "| train/              |          |\n",
-      "|    learning_rate    | 0.0001   |\n",
-      "|    loss             | 0.00122  |\n",
-      "|    n_updates        | 13998    |\n",
-      "----------------------------------\n",
-      "----------------------------------\n",
-      "| rollout/            |          |\n",
-      "|    ep_len_mean      | 49.9     |\n",
-      "|    ep_rew_mean      | 3.65     |\n",
-      "|    exploration_rate | 0.05     |\n",
-      "| time/               |          |\n",
-      "|    episodes         | 1000     |\n",
-      "|    fps              | 4092     |\n",
+      "|    episodes         | 700      |\n",
+      "|    fps              | 4445     |\n",
       "|    time_elapsed     | 14       |\n",
-      "|    total_timesteps  | 61082    |\n",
+      "|    total_timesteps  | 62462    |\n",
       "| train/              |          |\n",
       "|    learning_rate    | 0.0001   |\n",
-      "|    loss             | 0.0014   |\n",
-      "|    n_updates        | 15245    |\n",
+      "|    loss             | 6.99e-05 |\n",
+      "|    n_updates        | 15590    |\n",
       "----------------------------------\n",
       "----------------------------------\n",
       "| rollout/            |          |\n",
-      "|    ep_len_mean      | 42.8     |\n",
-      "|    ep_rew_mean      | 16.8     |\n",
+      "|    ep_len_mean      | 75.5     |\n",
+      "|    ep_rew_mean      | -49.2    |\n",
       "|    exploration_rate | 0.05     |\n",
       "| time/               |          |\n",
-      "|    episodes         | 1100     |\n",
-      "|    fps              | 4106     |\n",
+      "|    episodes         | 800      |\n",
+      "|    fps              | 4456     |\n",
       "|    time_elapsed     | 15       |\n",
-      "|    total_timesteps  | 65360    |\n",
+      "|    total_timesteps  | 70012    |\n",
       "| train/              |          |\n",
       "|    learning_rate    | 0.0001   |\n",
-      "|    loss             | 0.0115   |\n",
-      "|    n_updates        | 16314    |\n",
+      "|    loss             | 0.264    |\n",
+      "|    n_updates        | 17477    |\n",
       "----------------------------------\n",
       "----------------------------------\n",
       "| rollout/            |          |\n",
-      "|    ep_len_mean      | 30.9     |\n",
-      "|    ep_rew_mean      | 40.9     |\n",
+      "|    ep_len_mean      | 70.5     |\n",
+      "|    ep_rew_mean      | -39.2    |\n",
       "|    exploration_rate | 0.05     |\n",
       "| time/               |          |\n",
-      "|    episodes         | 1200     |\n",
-      "|    fps              | 4113     |\n",
-      "|    time_elapsed     | 16       |\n",
-      "|    total_timesteps  | 68446    |\n",
-      "| train/              |          |\n",
-      "|    learning_rate    | 0.0001   |\n",
-      "|    loss             | 0.00337  |\n",
-      "|    n_updates        | 17086    |\n",
-      "----------------------------------\n",
-      "----------------------------------\n",
-      "| rollout/            |          |\n",
-      "|    ep_len_mean      | 35       |\n",
-      "|    ep_rew_mean      | 33.7     |\n",
-      "|    exploration_rate | 0.05     |\n",
-      "| time/               |          |\n",
-      "|    episodes         | 1300     |\n",
-      "|    fps              | 4122     |\n",
+      "|    episodes         | 900      |\n",
+      "|    fps              | 4471     |\n",
       "|    time_elapsed     | 17       |\n",
-      "|    total_timesteps  | 71948    |\n",
-      "| train/              |          |\n",
-      "|    learning_rate    | 0.0001   |\n",
-      "|    loss             | 0.039    |\n",
-      "|    n_updates        | 17961    |\n",
-      "----------------------------------\n",
-      "----------------------------------\n",
-      "| rollout/            |          |\n",
-      "|    ep_len_mean      | 31.7     |\n",
-      "|    ep_rew_mean      | 39       |\n",
-      "|    exploration_rate | 0.05     |\n",
-      "| time/               |          |\n",
-      "|    episodes         | 1400     |\n",
-      "|    fps              | 4128     |\n",
-      "|    time_elapsed     | 18       |\n",
-      "|    total_timesteps  | 75122    |\n",
+      "|    total_timesteps  | 77066    |\n",
       "| train/              |          |\n",
       "|    learning_rate    | 0.0001   |\n",
-      "|    loss             | 0.00402  |\n",
-      "|    n_updates        | 18755    |\n",
+      "|    loss             | 0.000102 |\n",
+      "|    n_updates        | 19241    |\n",
       "----------------------------------\n",
       "----------------------------------\n",
       "| rollout/            |          |\n",
-      "|    ep_len_mean      | 27.7     |\n",
-      "|    ep_rew_mean      | 47       |\n",
+      "|    ep_len_mean      | 66.1     |\n",
+      "|    ep_rew_mean      | -28.8    |\n",
       "|    exploration_rate | 0.05     |\n",
       "| time/               |          |\n",
-      "|    episodes         | 1500     |\n",
-      "|    fps              | 4129     |\n",
+      "|    episodes         | 1000     |\n",
+      "|    fps              | 4489     |\n",
       "|    time_elapsed     | 18       |\n",
-      "|    total_timesteps  | 77894    |\n",
-      "| train/              |          |\n",
-      "|    learning_rate    | 0.0001   |\n",
-      "|    loss             | 0.00105  |\n",
-      "|    n_updates        | 19448    |\n",
-      "----------------------------------\n",
-      "----------------------------------\n",
-      "| rollout/            |          |\n",
-      "|    ep_len_mean      | 35.5     |\n",
-      "|    ep_rew_mean      | 31.2     |\n",
-      "|    exploration_rate | 0.05     |\n",
-      "| time/               |          |\n",
-      "|    episodes         | 1600     |\n",
-      "|    fps              | 4125     |\n",
-      "|    time_elapsed     | 19       |\n",
-      "|    total_timesteps  | 81440    |\n",
+      "|    total_timesteps  | 83678    |\n",
       "| train/              |          |\n",
       "|    learning_rate    | 0.0001   |\n",
-      "|    loss             | 0.00372  |\n",
-      "|    n_updates        | 20334    |\n",
+      "|    loss             | 0.000145 |\n",
+      "|    n_updates        | 20894    |\n",
       "----------------------------------\n",
       "----------------------------------\n",
       "| rollout/            |          |\n",
-      "|    ep_len_mean      | 27.9     |\n",
-      "|    ep_rew_mean      | 46.8     |\n",
+      "|    ep_len_mean      | 66.9     |\n",
+      "|    ep_rew_mean      | -31.6    |\n",
       "|    exploration_rate | 0.05     |\n",
       "| time/               |          |\n",
-      "|    episodes         | 1700     |\n",
-      "|    fps              | 4122     |\n",
+      "|    episodes         | 1100     |\n",
+      "|    fps              | 4504     |\n",
       "|    time_elapsed     | 20       |\n",
-      "|    total_timesteps  | 84230    |\n",
-      "| train/              |          |\n",
-      "|    learning_rate    | 0.0001   |\n",
-      "|    loss             | 0.011    |\n",
-      "|    n_updates        | 21032    |\n",
-      "----------------------------------\n",
-      "----------------------------------\n",
-      "| rollout/            |          |\n",
-      "|    ep_len_mean      | 34.3     |\n",
-      "|    ep_rew_mean      | 33.4     |\n",
-      "|    exploration_rate | 0.05     |\n",
-      "| time/               |          |\n",
-      "|    episodes         | 1800     |\n",
-      "|    fps              | 4122     |\n",
-      "|    time_elapsed     | 21       |\n",
-      "|    total_timesteps  | 87656    |\n",
+      "|    total_timesteps  | 90370    |\n",
       "| train/              |          |\n",
       "|    learning_rate    | 0.0001   |\n",
-      "|    loss             | 0.00412  |\n",
-      "|    n_updates        | 21888    |\n",
+      "|    loss             | 0.000488 |\n",
+      "|    n_updates        | 22567    |\n",
       "----------------------------------\n",
       "----------------------------------\n",
       "| rollout/            |          |\n",
-      "|    ep_len_mean      | 27.3     |\n",
-      "|    ep_rew_mean      | 48.5     |\n",
+      "|    ep_len_mean      | 68.6     |\n",
+      "|    ep_rew_mean      | -34.3    |\n",
       "|    exploration_rate | 0.05     |\n",
       "| time/               |          |\n",
-      "|    episodes         | 1900     |\n",
-      "|    fps              | 4122     |\n",
+      "|    episodes         | 1200     |\n",
+      "|    fps              | 4517     |\n",
       "|    time_elapsed     | 21       |\n",
-      "|    total_timesteps  | 90384    |\n",
+      "|    total_timesteps  | 97230    |\n",
       "| train/              |          |\n",
       "|    learning_rate    | 0.0001   |\n",
-      "|    loss             | 6.7      |\n",
-      "|    n_updates        | 22570    |\n",
-      "----------------------------------\n",
-      "----------------------------------\n",
-      "| rollout/            |          |\n",
-      "|    ep_len_mean      | 35.2     |\n",
-      "|    ep_rew_mean      | 31.5     |\n",
-      "|    exploration_rate | 0.05     |\n",
-      "| time/               |          |\n",
-      "|    episodes         | 2000     |\n",
-      "|    fps              | 4106     |\n",
-      "|    time_elapsed     | 22       |\n",
-      "|    total_timesteps  | 93900    |\n",
-      "| train/              |          |\n",
-      "|    learning_rate    | 0.0001   |\n",
-      "|    loss             | 0.0141   |\n",
-      "|    n_updates        | 23449    |\n",
-      "----------------------------------\n",
-      "----------------------------------\n",
-      "| rollout/            |          |\n",
-      "|    ep_len_mean      | 25.8     |\n",
-      "|    ep_rew_mean      | 51       |\n",
-      "|    exploration_rate | 0.05     |\n",
-      "| time/               |          |\n",
-      "|    episodes         | 2100     |\n",
-      "|    fps              | 4105     |\n",
-      "|    time_elapsed     | 23       |\n",
-      "|    total_timesteps  | 96476    |\n",
-      "| train/              |          |\n",
-      "|    learning_rate    | 0.0001   |\n",
-      "|    loss             | 0.0174   |\n",
-      "|    n_updates        | 24093    |\n",
-      "----------------------------------\n",
-      "----------------------------------\n",
-      "| rollout/            |          |\n",
-      "|    ep_len_mean      | 27.7     |\n",
-      "|    ep_rew_mean      | 47       |\n",
-      "|    exploration_rate | 0.05     |\n",
-      "| time/               |          |\n",
-      "|    episodes         | 2200     |\n",
-      "|    fps              | 4110     |\n",
-      "|    time_elapsed     | 24       |\n",
-      "|    total_timesteps  | 99250    |\n",
-      "| train/              |          |\n",
-      "|    learning_rate    | 0.0001   |\n",
-      "|    loss             | 6.27     |\n",
-      "|    n_updates        | 24787    |\n",
+      "|    loss             | 0.00045  |\n",
+      "|    n_updates        | 24282    |\n",
       "----------------------------------\n"
      ]
     }
@@ -608,13 +462,12 @@
     "env = Cube2()\n",
     "env = RewardWrapper(env)\n",
     "model = DQN(\"MlpPolicy\", env, verbose=1)\n",
-    "model.learn(total_timesteps=100000, log_interval=100)\n",
-    "model.save(\"dqn_cube2.pkl\")"
+    "model.learn(total_timesteps=100000, log_interval=100)"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 148,
+   "execution_count": 75,
    "id": "24132717",
    "metadata": {},
    "outputs": [
@@ -622,10 +475,10 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "rotationController.setState([[4.0, 4.0, 0.0, 0.0], [1.0, 5.0, 1.0, 5.0], [4.0, 2.0, 1.0, 2.0], [5.0, 3.0, 0.0, 3.0], [3.0, 1.0, 3.0, 4.0], [2.0, 0.0, 2.0, 5.0]])\n",
-      "rotationController.addRotationStepCode(...[7, 10, 2, 5, 7, 4, 10, 3, 11, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4])\n",
+      "rotationController.setState([[0.0, 0.0, 3.0, 4.0], [5.0, 2.0, 1.0, 1.0], [3.0, 4.0, 3.0, 2.0], [2.0, 5.0, 4.0, 5.0], [0.0, 3.0, 5.0, 1.0], [1.0, 2.0, 4.0, 0.0]])\n",
+      "rotationController.addRotationStepCode(...[3, 1, 8, 3])\n",
       "\n",
-      "Solved in 98 steps\n"
+      "Solved in 4 steps\n"
      ]
     }
    ],
@@ -650,14 +503,6 @@
     "print()\n",
     "print(f\"Solved in {len(solved_actions)} steps\")\n"
    ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "39924b6b",
-   "metadata": {},
-   "outputs": [],
-   "source": []
   }
  ],
  "metadata": {