update environments to use RLCore

Author: findmyway

Manifest.toml

@@ -7,8 +7,8 @@ version = "0.2.0"
 Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f"
 Plots = "91a5bcdd-55d7-5caf-9e0b-520d859cae80"
 Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
-ReinforcementLearning = "158674fc-8238-5cab-b5ba-03dfc80d1318"
-ReinforcementLearningEnvironments = "25e41dd2-4622-11e9-1641-f1adca772921"
+ReinforcementLearningBase = "e575027e-6cd6-5018-9292-cdc6200d2b44"
+ReinforcementLearningCore = "de1b191a-4ae0-4afa-a27b-92d07f46b2d6"
 SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf"
 StatsBase = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91"
 StatsPlots = "f3b207a7-027a-5e70-b257-86293d7955fd"

Project.toml

@@ -101,7 +101,7 @@
     {
      "data": {
       "text/plain": [
-       "Observation{Float64,Bool,Int64,NamedTuple{(:legal_actions,),Tuple{Array{Bool,1}}}}(0.0, false, 4186, (legal_actions = Bool[1, 1, 1, 1, 1, 1, 1, 1, 1, 0],))"
+       "{Float64,Bool,Int64,NamedTuple{(:legal_actions,),Tuple{Array{Bool,1}}}}(0.0, false, 4186, (legal_actions = Bool[1, 1, 1, 1, 1, 1, 1, 1, 1, 0],))"
       ]
      },
      "execution_count": 4,

notebooks/Chapter01_Tic_Tac_Toe.ipynb

@@ -115,7 +115,7 @@
     {
      "data": {
       "text/plain": [
-       "Observation{Float64,Bool,Int64,NamedTuple{(),Tuple{}}}(0.0, false, 501, NamedTuple())"
+       "{Float64,Bool,Int64,NamedTuple{(),Tuple{}}}(0.0, false, 501, NamedTuple())"
       ]
      },
      "execution_count": 6,

notebooks/Chapter09_Random_Walk.ipynb

@@ -1,9 +1,9 @@
 module AccessControl
 
-export AccessControlEnv, reset!, observe, interact!
+export AccessControlEnv
+
+using ReinforcementLearningCore
 
-using ReinforcementLearningEnvironments
-import ReinforcementLearningEnvironments: reset!, observe, interact!
 
 using Distributions
 
@@ -16,25 +16,17 @@ const CUSTOMERS = 1:length(PRIORITIES)
 
 const TRANSFORMER = LinearIndices((0:N_SERVERS, CUSTOMERS))
 
-mutable struct AccessControlEnv <: AbstractEnv
-    n_servers::Int
-    n_free_servers::Int
-    customer::Int
-    reward::Float64
-    observation_space::DiscreteSpace
-    action_space::DiscreteSpace
-    AccessControlEnv() =
-        new(
-            10,
-            0,
-            rand(CUSTOMERS),
-            0.0,
-            DiscreteSpace(length(TRANSFORMER)),
-            DiscreteSpace(2),
-        )
+Base.@kwdef mutable struct AccessControlEnv <: AbstractEnv
+    n_servers::Int = 10
+    n_free_servers::Int = 0
+    customer::Int = rand(CUSTOMERS)
+    reward::Float64 = 0.0
 end
 
-function interact!(env::AccessControlEnv, a)
+RLBase.get_observation_space(env::AccessControlEnv) = DiscreteSpace(length(TRANSFORMER))
+RLBase.get_action_space(env::AccessControlEnv) = DiscreteSpace(2)
+
+function (env::AccessControlEnv)(a)
     action, reward = ACTIONS[a], 0.0
     if env.n_free_servers > 0 && action == :accept
         env.n_free_servers -= 1
@@ -48,14 +40,14 @@ function interact!(env::AccessControlEnv, a)
     nothing
 end
 
-observe(env::AccessControlEnv) =
-    Observation(
+RLBase.observe(env::AccessControlEnv) =
+    (
         reward = env.reward,
         terminal = false,
         state = TRANSFORMER[CartesianIndex(env.n_free_servers + 1, env.customer)],
     )
 
-function reset!(env::AccessControlEnv)
+function RLBase.reset!(env::AccessControlEnv)
     env.n_free_servers = env.n_servers
     env.customer = rand(CUSTOMERS)
     env.reward = 0.0

src/environments/AccessControl.jl

@@ -1,20 +1,19 @@
 module BairdCounter
 
-export BairdCounterEnv, reset!, observe, interact!
+export BairdCounterEnv
 
-using ReinforcementLearningEnvironments
-import ReinforcementLearningEnvironments: reset!, observe, interact!
+using ReinforcementLearningCore
 
 const ACTIONS = (:dashed, :solid)
 
-mutable struct BairdCounterEnv <: AbstractEnv
-    current::Int
-    observation_space::DiscreteSpace
-    action_space::DiscreteSpace
-    BairdCounterEnv() = new(rand(1:7), DiscreteSpace(7), DiscreteSpace(length(ACTIONS)))
+Base.@kwdef mutable struct BairdCounterEnv <: AbstractEnv
+    current::Int = rand(1:7)
 end
 
-function interact!(env::BairdCounterEnv, a)
+RLBase.get_observation_space(env::BairdCounterEnv) = DiscreteSpace(7)
+RLBase.get_action_space(env::BairdCounterEnv) = DiscreteSpace(length(ACTIONS))
+
+function (env::BairdCounterEnv)(a)
     if ACTIONS[a] == :dashed
         env.current = rand(1:6)
     else
@@ -23,10 +22,10 @@ function interact!(env::BairdCounterEnv, a)
     nothing
 end
 
-observe(env::BairdCounterEnv) =
-    Observation(reward = 0.0, terminal = false, state = env.current)
+RLBase.observe(env::BairdCounterEnv) =
+    (reward = 0.0, terminal = false, state = env.current)
 
-function reset!(env::BairdCounterEnv)
+function RLBase.reset!(env::BairdCounterEnv)
     env.current = rand(1:6)
     nothing
 end

src/environments/BairdCounter.jl

@@ -1,10 +1,8 @@
 module BlackJack
 
-export BlackJackEnv, reset!, observe, interact!
-
-using ReinforcementLearningEnvironments
-import ReinforcementLearningEnvironments: reset!, observe, interact!
+export BlackJackEnv
 
+using ReinforcementLearningCore
 using Random
 
 const ACTIONS = [:hit, :stick]
@@ -48,10 +46,11 @@ mutable struct BlackJackEnv <: AbstractEnv
     reward::Float64
     is_exploring_start::Bool
     init::Union{Nothing,Tuple{Hands,Hands}}
-    observation_space::DiscreteSpace
-    action_space::DiscreteSpace
 end
 
+RLBase.get_observation_space(env::BlackJackEnv) = DiscreteSpace(length(INDS))
+RLBase.get_action_space(env::BlackJackEnv) = DiscreteSpace(2)
+
 function BlackJackEnv(; is_exploring_start = false, init = nothing)
     env = BlackJackEnv(
         Hands(),
@@ -60,8 +59,6 @@ function BlackJackEnv(; is_exploring_start = false, init = nothing)
         0.0,
         is_exploring_start,
         init,
-        DiscreteSpace(length(INDS)),
-        DiscreteSpace(2),
     )
     init_hands!(env)
     env
@@ -87,7 +84,7 @@ function init_hands!(env::BlackJackEnv)
     env.player_hands, env.dealer_hands = player_hands, dealer_hands
 end
 
-function interact!(env::BlackJackEnv, a::Int)
+function (env::BlackJackEnv)(a::Int)
     if ACTIONS[a] == :hit
         push!(env.player_hands, deal_card())
         if is_busted(env.player_hands)
@@ -117,7 +114,7 @@ function interact!(env::BlackJackEnv, a::Int)
     nothing
 end
 
-function reset!(env::BlackJackEnv)
+function RLBase.reset!(env::BlackJackEnv)
     env.is_end = false
     env.reward = 0.0
 
@@ -133,7 +130,7 @@ encode(env) =
         2 <= env.dealer_hands.sum <= 10 ? env.dealer_hands.sum : 1,
     ]
 
-observe(env::BlackJackEnv) =
-    Observation(reward = env.reward, terminal = env.is_end, state = encode(env))
+RLBase.observe(env::BlackJackEnv) =
+    (reward = env.reward, terminal = env.is_end, state = encode(env))
 
 end

src/environments/BlackJack.jl

@@ -1,9 +1,8 @@
 module BranchMDP
 
-export BranchMDPEnv, reset!, observe, interact!
+export BranchMDPEnv
 
-using ReinforcementLearningEnvironments
-import ReinforcementLearningEnvironments: reset!, observe, interact!
+using ReinforcementLearningCore
 
 mutable struct BranchMDPEnv <: AbstractEnv
     transition::Array{Int,3}
@@ -26,7 +25,10 @@ mutable struct BranchMDPEnv <: AbstractEnv
         )
 end
 
-function interact!(env::BranchMDPEnv, a::Int)
+RLBase.get_observation_space(env::BranchMDPEnv) = env.observation_space
+RLBase.get_action_space(env::BranchMDPEnv) = env.action_space
+
+function (env::BranchMDPEnv)(a::Int)
     if rand() < env.termination_prob
         env.reward = 0.0
         env.current = size(env.transition, 1) + 1
@@ -40,14 +42,14 @@ function interact!(env::BranchMDPEnv, a::Int)
     nothing
 end
 
-observe(env::BranchMDPEnv) =
-    Observation(
+RLBase.observe(env::BranchMDPEnv) =
+    (
         reward = env.reward,
         terminal = env.current == size(env.transition, 1) + 1,
         state = env.current,
     )
 
-function reset!(env::BranchMDPEnv, s::Int = 1)
+function RLBase.reset!(env::BranchMDPEnv, s::Int = 1)
     env.current = s
     nothing
 end

src/environments/BranchMDP.jl

@@ -1,9 +1,9 @@
 module CliffWalking
 
-export CliffWalkingEnv, reset!, observe, interact!
+export CliffWalkingEnv
+
+using ReinforcementLearningCore
 
-using ReinforcementLearningEnvironments
-import ReinforcementLearningEnvironments: reset!, observe, interact!
 
 const NX = 4
 const NY = 12
@@ -23,28 +23,27 @@ function iscliff(p::CartesianIndex{2})
     x == 4 && y > 1 && y < NY
 end
 
-mutable struct CliffWalkingEnv <: AbstractEnv
-    position::CartesianIndex{2}
-    observation_space::DiscreteSpace
-    action_space::DiscreteSpace
-    CliffWalkingEnv() =
-        new(Start, DiscreteSpace(length(LinearInds)), DiscreteSpace(length(Actions)))
+Base.@kwdef mutable struct CliffWalkingEnv <: AbstractEnv
+    position::CartesianIndex{2} = Start
 end
 
-function interact!(env::CliffWalkingEnv, a::Int)
+RLBase.get_observation_space(env::CliffWalkingEnv) = DiscreteSpace(length(LinearInds))
+RLBase.get_action_space(env::CliffWalkingEnv) = DiscreteSpace(length(Actions))
+
+function (env::CliffWalkingEnv)(a::Int)
     x, y = Tuple(env.position + Actions[a])
     env.position = CartesianIndex(min(max(x, 1), NX), min(max(y, 1), NY))
     nothing
 end
 
-observe(env::CliffWalkingEnv) =
-    Observation(
+RLBase.observe(env::CliffWalkingEnv) =
+    (
         reward = env.position == Goal ? 0.0 : (iscliff(env.position) ? -100.0 : -1.0),
         terminal = env.position == Goal || iscliff(env.position),
         state = LinearInds[env.position],
     )
 
-function reset!(env::CliffWalkingEnv)
+function RLBase.reset!(env::CliffWalkingEnv)
     env.position = Start
     nothing
 end

src/environments/CliffWalking.jl

@@ -1,43 +1,42 @@
 module LeftRight
 
-export LeftRightEnv, reset!, observe, interact!
+export LeftRightEnv
+
+using ReinforcementLearningCore
 
-using ReinforcementLearningEnvironments
-import ReinforcementLearningEnvironments: reset!, observe, interact!
 
 using StatsBase
 
 mutable struct LeftRightEnv <: AbstractEnv
     transitions::Array{Float64,3}
     current_state::Int
-    observation_space::DiscreteSpace
-    action_space::DiscreteSpace
-    LeftRightEnv(transitions, current_state) =
-        new(transitions, current_state, DiscreteSpace(2), DiscreteSpace(2))
 end
 
+RLBase.get_observation_space(env::LeftRightEnv) = DiscreteSpace(2)
+RLBase.get_action_space(env::LeftRightEnv) = DiscreteSpace(2)
+
 function LeftRightEnv()
     t = zeros(2, 2, 2)
     t[1, :, :] = [0.9 0.1; 0.0 1.0]
     t[2, :, :] = [0.0 1.0; 0.0 1.0]
     LeftRightEnv(t, rand(1:2))
 end
 
-function interact!(env::LeftRightEnv, a::Int)
+function (env::LeftRightEnv)(a::Int)
     env.current_state = sample(Weights(
         @view(env.transitions[env.current_state, a, :]),
         1.0,
     ))
     nothing
 end
 
-function reset!(env::LeftRightEnv)
+function RLBase.reset!(env::LeftRightEnv)
     env.current_state = 1
     nothing
 end
 
-observe(env::LeftRightEnv) =
-    Observation(
+RLBase.observe(env::LeftRightEnv) =
+    (
         reward = Float64(env.current_state == 2),
         terminal = env.current_state == 2,
         state = env.current_state,