From a922607bcb3500d7e3747c01593deea6951fa067 Mon Sep 17 00:00:00 2001 From: pockerman Date: Fri, 6 Feb 2026 19:36:07 +0000 Subject: [PATCH] Update documentation --- examples/example_1/example_1.md | 10 +-- examples/example_2/example_2.cpp | 2 +- examples/example_2/example_2.md | 122 +++++++++++++++++++++++++++++++ examples/example_3/example_3.md | 88 ++++++++++++++++++++++ examples/example_4/example_4.md | 84 +++++++++++++++++++++ 5 files changed, 300 insertions(+), 6 deletions(-) create mode 100644 examples/example_2/example_2.md create mode 100644 examples/example_3/example_3.md create mode 100644 examples/example_4/example_4.md diff --git a/examples/example_1/example_1.md b/examples/example_1/example_1.md index f49c39e..95a4cc6 100644 --- a/examples/example_1/example_1.md +++ b/examples/example_1/example_1.md @@ -1,11 +1,11 @@ -\page bitrl_example_1 Example 1 Using Gymnasium environments +\page bitrl_example_1 BitRL Example 1 Using Gymnasium environments (Part 1) In this example we will see how to interact with Gymnasium environments and specifically how to create an interact with FroznLake environment. -In bitrl, Gymnasium-based environment are interacted over a REST-like API maintained here: bitrl-envs-api. -bitrl itself implements classes that hide this interaction from the client code. -In general, environment classes in bitrl, have to implement the \ref bitrl::envs::EnvBase "bitrl::envs::EnvBase" API. +In _bitrl_, Gymnasium-based environments are interacted over a REST-like API maintained here: bitrl-envs-api. +_bitrl_ itself implements classes that hide this interaction from the client code. +In general, environment classes in _bitrl_, have to implement the \ref bitrl::envs::EnvBase "bitrl::envs::EnvBase" API. In this example we will use the \ref bitrl::envs::gymnasium::FrozenLake "bitrl::envs::gymnasium::FrozenLake" class. This is a template class, see the example below, that itself inherits from \ref bitrl::envs::gymnasium::GymnasiumEnvBase "bitrl::envs::gymnasium::GymnasiumEnvBase" @@ -132,6 +132,6 @@ int main() @endcode In order to run the example you will need an instance of the bitrl-envs-api server running -on your machine listening at por 8001. Note the actual example also shows how to use \ref bitrl::envs::gymnasium::Taxi "bitrl::envs::gymnasium::Taxi", +on your machine listening at port 8001. Note the actual example also shows how to use \ref bitrl::envs::gymnasium::Taxi "bitrl::envs::gymnasium::Taxi", \ref bitrl::envs::gymnasium::CliffWorld "bitrl::envs::gymnasium::CliffWorld" and \ref bitrl::envs::gymnasium::BlackJack "bitrl::envs::gymnasium::BlackJack" environments. diff --git a/examples/example_2/example_2.cpp b/examples/example_2/example_2.cpp index 8c69075..4f9087f 100644 --- a/examples/example_2/example_2.cpp +++ b/examples/example_2/example_2.cpp @@ -43,7 +43,7 @@ void test_gymwalk(RESTRLEnvClient &server) // once the env is created we can get it's id std::cout << "Environment idx is: " << env.idx() << std::endl; - // the create flag should be true + // the created flag should be true std::cout << "Is environment created? " << env.is_created() << std::endl; // environment should be alive on the server diff --git a/examples/example_2/example_2.md b/examples/example_2/example_2.md new file mode 100644 index 0000000..1321c47 --- /dev/null +++ b/examples/example_2/example_2.md @@ -0,0 +1,122 @@ +\page bitrl_example_2 BitRL Example 2 Using GymWalk Environment + +The \ref bitrl::envs::gdrl::GymWalk "bitrl::envs::gdrl::GymWalk" is an implementation of the +environment from https://github.com/mimoralea/gym-walk. Just like with \ref bitrl_example_1, you will need +an instance of the bitrl-envs-api server running in order +to actually use the environment. Other than that there isn't much to say for this example. +Below is the driver code. + +@code{.cpp} +#include "bitrl/bitrl_types.h" +#include "bitrl/envs/gdrl/gym_walk.h" +#include "bitrl/network/rest_rl_env_client.h" + +#include +#include +#include +#include + +namespace example_1 +{ +using namespace bitrl; + +const std::string SERVER_URL = "http://0.0.0.0:8001/api"; + +using bitrl::envs::gdrl::GymWalk; +using bitrl::network::RESTRLEnvClient; + +void test_gymwalk(RESTRLEnvClient &server) +{ +// the environment is not registered with the server +std::cout << "Is environment registered: " << server.is_env_registered(GymWalk<4>::name) +<< std::endl; + + // when the environment is created we register it with the REST client + GymWalk<4> env(server); + + // environment name can also be accessed via env.env_name() + std::cout << "Is environment registered: " << server.is_env_registered(env.env_name()) + << std::endl; + std::cout << "Environment URL: " << env.get_url() << std::endl; + + // make the environment we pass both make options + // and reset options + std::unordered_map make_ops; + std::unordered_map reset_ops; + reset_ops.insert({"seed", static_cast(42)}); + env.make("v1", make_ops, reset_ops); + + // query the environemnt version + std::cout << "Environment version: " << env.version() << std::endl; + + // once the env is created we can get it's id + std::cout << "Environment idx is: " << env.idx() << std::endl; + + // the created flag should be true + std::cout << "Is environment created? " << env.is_created() << std::endl; + + // environment should be alive on the server + std::cout << "Is environment alive? " << env.is_alive() << std::endl; + + // FrozenLake is a discrete state-action env so we can + // query number of actions and states + std::cout << "Number of valid actions? " << env.n_actions() << std::endl; + std::cout << "Number of states? " << env.n_states() << std::endl; + + // reset the environment + auto time_step = env.reset(); + + std::cout << "Reward on reset: " << time_step.reward() << std::endl; + std::cout << "Observation on reset: " << time_step.observation() << std::endl; + std::cout << "Is terminal state: " << time_step.done() << std::endl; + + //...print the time_step + std::cout << time_step << std::endl; + + // take an action in the environment + // 2 = RIGHT + auto new_time_step = env.step(1); + std::cout << new_time_step << std::endl; + + // get the dynamics of the environment for the given state and action + auto state = 0; + auto action = 1; + auto dynamics = env.p(state, action); + + std::cout << "Dynamics for state=" << state << " and action=" << action << std::endl; + for (auto item : dynamics) + { + std::cout << std::get<0>(item) << std::endl; + std::cout << std::get<1>(item) << std::endl; + std::cout << std::get<2>(item) << std::endl; + std::cout << std::get<3>(item) << std::endl; + } + + // discrete action environments can sample + // actions + action = env.sample_action(); + std::cout << "Action sampled: " << action << std::endl; + + new_time_step = env.step(action); + std::cout << new_time_step << std::endl; + + // close the environment + env.close(); +} + +} // namespace example_1 + +int main() +{ + + using namespace example_1; + + RESTRLEnvClient server(SERVER_URL, false); + + std::cout << "Testing GymWalk..." << std::endl; + example_1::test_gymwalk(server); + std::cout << "====================" << std::endl; + return 0; +} + +@endcode diff --git a/examples/example_3/example_3.md b/examples/example_3/example_3.md new file mode 100644 index 0000000..91dc42f --- /dev/null +++ b/examples/example_3/example_3.md @@ -0,0 +1,88 @@ +\page bitrl_example_3 BitRL Example 3 Using Gymnasium environments (Part 2) + +In this example we will see how to interact with Gymnasium environments and +specifically how to create an interact with CartPole environment. + +As already mentioned in \ref bitrl_example_1, Gymnasium-based environments are interacted over a REST-like API maintained here: bitrl-envs-api. +_bitrl_ itself implements classes that hide this interaction from the client code. +In general, environment classes in _bitrl_, have to implement the \ref bitrl::envs::EnvBase "bitrl::envs::EnvBase" API. + +In this example we will use the \ref bitrl::envs::gymnasium::CartPole "bitrl::envs::gymnasium::CartPole" +class. This is a template class, see the example below, that itself inherits from \ref bitrl::envs::gymnasium::GymnasiumEnvBase "bitrl::envs::gymnasium::GymnasiumEnvBase" +class. + +Below is the driver code. + +@code{.cpp} + +#include "bitrl/bitrl_consts.h" +#include "bitrl/bitrl_types.h" +#include "bitrl/envs/gymnasium/classic_control/cart_pole_env.h" +#include "bitrl/network/rest_rl_env_client.h" + +#ifdef BITRL_DEBUG +#include +#endif + +#include +#include +#include +#include + +namespace example +{ +using namespace bitrl; +using namespace bitrl::envs::gymnasium; +using bitrl::network::RESTRLEnvClient; + +void test_cart_pole(RESTRLEnvClient &server) +{ +std::cout << "Is environment registered: " << server.is_env_registered(CartPole::name) +<< std::endl; + + // create the environment + CartPole env(server); + + std::cout << "Name: " << env.name << std::endl; + std::cout << "Number of actions: " << env.n_actions() << std::endl; + + // make the environment + std::unordered_map options; + std::unordered_map reset_ops; + reset_ops.insert({"seed", static_cast(42)}); + env.make("v1", options, reset_ops); + + auto time_step = env.reset(); + std::cout << "Time step: " << time_step << std::endl; + + // step in the environment + time_step = env.step(0); + std::cout << "Time step after action: " << time_step << std::endl; + + env.close(); +} + +} // namespace example + +int main() +{ + + using namespace example; + + const std::string SERVER_URL = "http://0.0.0.0:8001/api"; + + RESTRLEnvClient server(SERVER_URL, true); + + std::cout << "Testing CartPole..." << std::endl; + example::test_cart_pole(server); + std::cout << "====================" << std::endl; + + return 0; +} +@endcode + + +In order to run the example you will need an instance of the bitrl-envs-api server running +on your machine listening at port 8001. Note the actual example also shows how to use \ref bitrl::envs::gymnasium::Pendulum "bitrl::envs::gymnasium::Pendulum", +\ref bitrl::envs::gymnasium::Acrobot "bitrl::envs::gymnasium::Acrobot" and \ref bitrl::envs::gymnasium::MountainCar "bitrl::envs::gymnasium::MountainCar" +environments. \ No newline at end of file diff --git a/examples/example_4/example_4.md b/examples/example_4/example_4.md new file mode 100644 index 0000000..116ce0f --- /dev/null +++ b/examples/example_4/example_4.md @@ -0,0 +1,84 @@ +\page bitrl_example_4 BitRL Example 4 Using Gymnasium environments (Part 3) + +This example follows \ref bitrl_example_1 and \ref bitrl_example_3 and shows how to use +\ref bitrl::envs::gymnasium::LunarLanderDiscreteEnv "bitrl::envs::gymnasium::LunarLanderDiscreteEnv" and +\ref bitrl::envs::gymnasium::LunarLanderContinuousEnv "bitrl::envs::gymnasium::LunarLanderContinuousEnv". + + +@code{.cpp} + +#include "bitrl/envs/gymnasium/box2d/lunar_lander_env.h" +#include "bitrl/network/rest_rl_env_client.h" + +#include +#include +#include + +namespace box2d_example +{ +using namespace bitrl; +const std::string SERVER_URL = "http://0.0.0.0:8001/api"; +using bitrl::real_t; +using bitrl::envs::gymnasium::LunarLanderContinuousEnv; +using bitrl::envs::gymnasium::LunarLanderDiscreteEnv; +} // namespace box2d_example + +int main() +{ +using namespace box2d_example; + + bitrl::network::RESTRLEnvClient server(SERVER_URL, true); + + std::unordered_map options; + options["wind_power"] = std::any(static_cast(10.0)); + options["enable_wind"] = std::any(static_cast(true)); + options["gravity"] = std::any(static_cast(-9.86)); + options["turbulence_power"] = std::any(static_cast(1.5)); + + std::unordered_map reset_ops; + reset_ops.insert({"seed", static_cast(42)}); + + { + std::cout<<"Working with LunarLanderDiscreteEnv..."< action = {0.8, 0.9}; + time_step = env.step(action); + std::cout << "Time step: " << time_step << std::endl; + env.close(); + } + + return 0; +} + +@endcode \ No newline at end of file