sync

Author: findmyway

notebooks/Chapter08_Maze.ipynb

@@ -0,0 +1 @@
+include("linear_approximator.jl")

notebooks/Chapter09_Random_Walk.ipynb

@@ -0,0 +1,50 @@
+export LinearVApproximator, LinearQApproximator
+
+using LinearAlgebra: dot
+
+"""
+    LinearVApproximator(weights::Array{Float64, N}) -> LinearVApproximator{N}
+Use the weighted sum to represent the estimation of a state.
+The state is expected to have the same length with `weights`.
+See also [`LinearQApproximator`](@ref)
+"""
+struct LinearVApproximator{N} <: AbstractApproximator
+    weights::Array{Float64,N}
+end
+
+RLBase.ApproximatorStyle(::LinearVApproximator) = VApproximator()
+
+# TODO: support Vector
+(V::LinearVApproximator)(s) = dot(s, V.weights)
+
+function RLBase.update!(V::LinearVApproximator, correction::Pair)
+    s, e = correction
+    V.weights .+= s .* e
+end
+
+"""
+    LinearQApproximator(weights::Vector{Float64}, feature_func::F, actions::Vector{Int}) -> LinearQApproximator{F}
+Use weighted sum to represent the estimation given a state and an action.
+# Fields
+- `weights::Vector{Float64}`: the weight of each feature.
+- `feature_func::Function`: decide how to generate a feature vector of `length(weights)` given a state and an action as parameters.
+- `actions::Vector{Int}`: all possible actions.
+See also [`LinearVApproximator`](@ref).
+"""
+Base.@kwdef struct LinearQApproximator{F} <: AbstractApproximator
+    weights::Vector{Float64}
+    feature_func::F
+    actions::Vector{Int}
+end
+
+RLBase.ApproximatorStyle(::LinearQApproximator) = QApproximator()
+
+(Q::LinearQApproximator)(s, a::Int) = dot(Q.weights, Q.feature_func(s, a))
+
+(Q::LinearQApproximator)(s) = [Q(s, a) for a in Q.actions]
+
+function RLBase.update!(Q::LinearQApproximator, correction::Pair)
+    (s, a), e = correction
+    xs = Q.feature_func(s, a)
+    Q.weights .+= xs .* e
+end

src/extensions/approximators/approximators.jl

@@ -1,4 +1,6 @@
 include("utils/utils.jl")
+include("preprocessors.jl")
+include("approximators/approximators.jl")
 include("environment_models/environment_models.jl")
 include("learners/learners.jl")
 include("policies/policies.jl")

src/extensions/approximators/linear_approximator.jl

@@ -0,0 +1,31 @@
+export FourierPreprocessor, PolynomialPreprocessor, TilingPreprocessor
+
+"""
+    FourierPreprocessor(order::Int)
+Transform a scalar to a vector of `order+1` Fourier bases.
+"""
+struct FourierPreprocessor <: AbstractPreprocessor
+    order::Int
+end
+
+(p::FourierPreprocessor)(s::Number) = [cos(i * π * s) for i = 0:p.order]
+
+"""
+    PolynomialPreprocessor(order::Int)
+Transform a scalar to vector of maximum `order` polynomial.
+"""
+struct PolynomialPreprocessor <: AbstractPreprocessor
+    order::Int
+end
+
+(p::PolynomialPreprocessor)(s::Number) = [s^i for i = 0:p.order]
+
+"""
+    TilingPreprocessor(tilings::Vector{<:Tiling})
+Use each `tilings` to encode the state and return a vector.
+"""
+struct TilingPreprocessor{Tt<:Tiling} <: AbstractPreprocessor
+    tilings::Vector{Tt}
+end
+
+(p::TilingPreprocessor)(s::Union{<:Number,<:Array}) = [encode(t, s) for t in p.tilings]

src/extensions/extensions.jl

@@ -59,4 +59,5 @@ end
 
 discount_rewards(rewards, γ) = discount_rewards!(similar(rewards), rewards, γ)
 
-discount_rewards_reduced(rewards, γ) = foldr((r, g) -> r + γ * g, rewards)
+discount_rewards_reduced(rewards, γ) = foldr((r, g) -> r + γ * g, rewards)
+

src/extensions/preprocessors.jl

@@ -0,0 +1,51 @@
+export Tiling, encode
+
+import Base: length, -
+
+"""
+    Tiling(ranges::NTuple{N, Tr}) where {N, Tr}
+Using a tuple of `ranges` to simulate a tiling.
+The length of `ranges` indicates the dimension of tilling.
+# Example
+```julia
+julia> t = Tiling((1:2:5, 10:5:20))
+Tiling{2,StepRange{Int64,Int64}}((1:2:5, 10:5:20), [1 3; 2 4])
+julia> encode(t, (2, 12))  # encode into an Int
+1
+julia> encode(t, (2, 18))
+3
+julia> t2 = t - (1, 3)  # shift a little to get a new Tiling
+Tiling{2,StepRange{Int64,Int64}}((0:2:4, 7:5:17), [1 3; 2 4])
+```
+"""
+struct Tiling{N,Tr<:AbstractRange}
+    ranges::NTuple{N,Tr}
+    inds::LinearIndices{N,NTuple{N,Base.OneTo{Int}}}
+    Tiling(ranges::NTuple{N,Tr}) where {N,Tr} =
+        new{N,Tr}(ranges, LinearIndices(Tuple(length(r) - 1 for r in ranges)))
+end
+
+"""
+    (-)(t::Tiling, xs)
+Shift `t` along each dimension by each element in `xs`.
+"""
+function Base.:-(t::Tiling, xs)
+    Tiling(Tuple(r .- x for (r, x) in zip(t.ranges, xs)))
+end
+
+Base.length(t::Tiling) = reduce(*, (length(r) - 1 for r in t.ranges))
+
+encode(range::AbstractRange, x) = floor(Int, div(x - range[1], step(range)) + 1)
+
+
+# TODO: use @generator here!
+encode(t::Tiling{1}, x::Number) = encode(t.ranges[1], x)
+encode(t::Tiling{1}, xs) = encode(t.ranges[1], xs[1])
+encode(t::Tiling{2}, xs) =
+    t.inds[CartesianIndex(encode(t.ranges[1], xs[1]), encode(t.ranges[2], xs[2]))]
+encode(t::Tiling{3}, xs) =
+    t.inds[CartesianIndex(
+        encode(t.ranges[1], xs[1]),
+        encode(t.ranges[2], xs[2]),
+        encode(t.ranges[3], xs[3]),
+    )]

src/extensions/utils/base.jl

@@ -1,2 +1,3 @@
 include("base.jl")
-include("optimizers.jl")
+include("optimizers.jl")
+include("tiling.jl")