Skip to content

Commit 42f574c

Browse files
findmywayfindmyway
committed
update
1 parent ddf9b6d commit 42f574c

3 files changed

Lines changed: 133 additions & 131 deletions

File tree

‎notebooks/Chapter04_Gambler_Problem.ipynb‎

Lines changed: 59 additions & 58 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,8 @@
66
"metadata": {},
77
"outputs": [],
88
"source": [
9-
"using ReinforcementLearning, ReinforcementLearningEnvironments"
9+
"using ReinforcementLearningCore\n",
10+
"using RLIntro"
1011
]
1112
},
1213
{
@@ -51,123 +52,123 @@
5152
"<?xml version=\"1.0\" encoding=\"utf-8\"?>\n",
5253
"<svg xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\" width=\"600\" height=\"400\" viewBox=\"0 0 2400 1600\">\n",
5354
"<defs>\n",
54-
" <clipPath id=\"clip3200\">\n",
55+
" <clipPath id=\"clip3300\">\n",
5556
" <rect x=\"0\" y=\"0\" width=\"2400\" height=\"1600\"/>\n",
5657
" </clipPath>\n",
5758
"</defs>\n",
58-
"<polygon clip-path=\"url(#clip3200)\" points=\"\n",
59-
"0,1600 2400,1600 2400,0 0,0 \n",
59+
"<path clip-path=\"url(#clip3300)\" d=\"\n",
60+
"M0 1600 L2400 1600 L2400 0 L0 0 Z\n",
6061
" \" fill=\"#ffffff\" fill-rule=\"evenodd\" fill-opacity=\"1\"/>\n",
6162
"<defs>\n",
62-
" <clipPath id=\"clip3201\">\n",
63+
" <clipPath id=\"clip3301\">\n",
6364
" <rect x=\"480\" y=\"0\" width=\"1681\" height=\"1600\"/>\n",
6465
" </clipPath>\n",
6566
"</defs>\n",
66-
"<polygon clip-path=\"url(#clip3200)\" points=\"\n",
67-
"153.898,1487.47 2352.76,1487.47 2352.76,47.2441 153.898,47.2441 \n",
67+
"<path clip-path=\"url(#clip3300)\" d=\"\n",
68+
"M153.898 1487.47 L2352.76 1487.47 L2352.76 47.2441 L153.898 47.2441 Z\n",
6869
" \" fill=\"#ffffff\" fill-rule=\"evenodd\" fill-opacity=\"1\"/>\n",
6970
"<defs>\n",
70-
" <clipPath id=\"clip3202\">\n",
71+
" <clipPath id=\"clip3302\">\n",
7172
" <rect x=\"153\" y=\"47\" width=\"2200\" height=\"1441\"/>\n",
7273
" </clipPath>\n",
7374
"</defs>\n",
74-
"<polyline clip-path=\"url(#clip3202)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
75+
"<polyline clip-path=\"url(#clip3302)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
7576
" 194.963,1487.47 194.963,47.2441 \n",
7677
" \"/>\n",
77-
"<polyline clip-path=\"url(#clip3202)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
78+
"<polyline clip-path=\"url(#clip3302)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
7879
" 724.145,1487.47 724.145,47.2441 \n",
7980
" \"/>\n",
80-
"<polyline clip-path=\"url(#clip3202)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
81+
"<polyline clip-path=\"url(#clip3302)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
8182
" 1253.33,1487.47 1253.33,47.2441 \n",
8283
" \"/>\n",
83-
"<polyline clip-path=\"url(#clip3202)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
84+
"<polyline clip-path=\"url(#clip3302)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
8485
" 1782.51,1487.47 1782.51,47.2441 \n",
8586
" \"/>\n",
86-
"<polyline clip-path=\"url(#clip3202)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
87+
"<polyline clip-path=\"url(#clip3302)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
8788
" 2311.69,1487.47 2311.69,47.2441 \n",
8889
" \"/>\n",
89-
"<polyline clip-path=\"url(#clip3202)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
90+
"<polyline clip-path=\"url(#clip3302)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
9091
" 153.898,1449.58 2352.76,1449.58 \n",
9192
" \"/>\n",
92-
"<polyline clip-path=\"url(#clip3202)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
93+
"<polyline clip-path=\"url(#clip3302)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
9394
" 153.898,1167.19 2352.76,1167.19 \n",
9495
" \"/>\n",
95-
"<polyline clip-path=\"url(#clip3202)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
96+
"<polyline clip-path=\"url(#clip3302)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
9697
" 153.898,884.807 2352.76,884.807 \n",
9798
" \"/>\n",
98-
"<polyline clip-path=\"url(#clip3202)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
99+
"<polyline clip-path=\"url(#clip3302)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
99100
" 153.898,602.42 2352.76,602.42 \n",
100101
" \"/>\n",
101-
"<polyline clip-path=\"url(#clip3202)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
102+
"<polyline clip-path=\"url(#clip3302)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
102103
" 153.898,320.033 2352.76,320.033 \n",
103104
" \"/>\n",
104-
"<polyline clip-path=\"url(#clip3200)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
105+
"<polyline clip-path=\"url(#clip3300)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
105106
" 153.898,1487.47 2352.76,1487.47 \n",
106107
" \"/>\n",
107-
"<polyline clip-path=\"url(#clip3200)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
108+
"<polyline clip-path=\"url(#clip3300)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
108109
" 153.898,1487.47 153.898,47.2441 \n",
109110
" \"/>\n",
110-
"<polyline clip-path=\"url(#clip3200)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
111-
" 194.963,1487.47 194.963,1465.87 \n",
111+
"<polyline clip-path=\"url(#clip3300)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
112+
" 194.963,1487.47 194.963,1470.19 \n",
112113
" \"/>\n",
113-
"<polyline clip-path=\"url(#clip3200)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
114-
" 724.145,1487.47 724.145,1465.87 \n",
114+
"<polyline clip-path=\"url(#clip3300)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
115+
" 724.145,1487.47 724.145,1470.19 \n",
115116
" \"/>\n",
116-
"<polyline clip-path=\"url(#clip3200)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
117-
" 1253.33,1487.47 1253.33,1465.87 \n",
117+
"<polyline clip-path=\"url(#clip3300)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
118+
" 1253.33,1487.47 1253.33,1470.19 \n",
118119
" \"/>\n",
119-
"<polyline clip-path=\"url(#clip3200)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
120-
" 1782.51,1487.47 1782.51,1465.87 \n",
120+
"<polyline clip-path=\"url(#clip3300)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
121+
" 1782.51,1487.47 1782.51,1470.19 \n",
121122
" \"/>\n",
122-
"<polyline clip-path=\"url(#clip3200)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
123-
" 2311.69,1487.47 2311.69,1465.87 \n",
123+
"<polyline clip-path=\"url(#clip3300)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
124+
" 2311.69,1487.47 2311.69,1470.19 \n",
124125
" \"/>\n",
125-
"<polyline clip-path=\"url(#clip3200)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
126-
" 153.898,1449.58 186.881,1449.58 \n",
126+
"<polyline clip-path=\"url(#clip3300)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
127+
" 153.898,1449.58 180.284,1449.58 \n",
127128
" \"/>\n",
128-
"<polyline clip-path=\"url(#clip3200)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
129-
" 153.898,1167.19 186.881,1167.19 \n",
129+
"<polyline clip-path=\"url(#clip3300)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
130+
" 153.898,1167.19 180.284,1167.19 \n",
130131
" \"/>\n",
131-
"<polyline clip-path=\"url(#clip3200)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
132-
" 153.898,884.807 186.881,884.807 \n",
132+
"<polyline clip-path=\"url(#clip3300)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
133+
" 153.898,884.807 180.284,884.807 \n",
133134
" \"/>\n",
134-
"<polyline clip-path=\"url(#clip3200)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
135-
" 153.898,602.42 186.881,602.42 \n",
135+
"<polyline clip-path=\"url(#clip3300)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
136+
" 153.898,602.42 180.284,602.42 \n",
136137
" \"/>\n",
137-
"<polyline clip-path=\"url(#clip3200)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
138-
" 153.898,320.033 186.881,320.033 \n",
138+
"<polyline clip-path=\"url(#clip3300)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
139+
" 153.898,320.033 180.284,320.033 \n",
139140
" \"/>\n",
140-
"<g clip-path=\"url(#clip3200)\">\n",
141+
"<g clip-path=\"url(#clip3300)\">\n",
141142
"<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:middle;\" transform=\"rotate(0, 194.963, 1541.47)\" x=\"194.963\" y=\"1541.47\">0</text>\n",
142143
"</g>\n",
143-
"<g clip-path=\"url(#clip3200)\">\n",
144+
"<g clip-path=\"url(#clip3300)\">\n",
144145
"<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:middle;\" transform=\"rotate(0, 724.145, 1541.47)\" x=\"724.145\" y=\"1541.47\">25</text>\n",
145146
"</g>\n",
146-
"<g clip-path=\"url(#clip3200)\">\n",
147+
"<g clip-path=\"url(#clip3300)\">\n",
147148
"<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:middle;\" transform=\"rotate(0, 1253.33, 1541.47)\" x=\"1253.33\" y=\"1541.47\">50</text>\n",
148149
"</g>\n",
149-
"<g clip-path=\"url(#clip3200)\">\n",
150+
"<g clip-path=\"url(#clip3300)\">\n",
150151
"<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:middle;\" transform=\"rotate(0, 1782.51, 1541.47)\" x=\"1782.51\" y=\"1541.47\">75</text>\n",
151152
"</g>\n",
152-
"<g clip-path=\"url(#clip3200)\">\n",
153+
"<g clip-path=\"url(#clip3300)\">\n",
153154
"<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:middle;\" transform=\"rotate(0, 2311.69, 1541.47)\" x=\"2311.69\" y=\"1541.47\">100</text>\n",
154155
"</g>\n",
155-
"<g clip-path=\"url(#clip3200)\">\n",
156+
"<g clip-path=\"url(#clip3300)\">\n",
156157
"<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:end;\" transform=\"rotate(0, 129.898, 1467.08)\" x=\"129.898\" y=\"1467.08\">0.0</text>\n",
157158
"</g>\n",
158-
"<g clip-path=\"url(#clip3200)\">\n",
159+
"<g clip-path=\"url(#clip3300)\">\n",
159160
"<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:end;\" transform=\"rotate(0, 129.898, 1184.69)\" x=\"129.898\" y=\"1184.69\">0.2</text>\n",
160161
"</g>\n",
161-
"<g clip-path=\"url(#clip3200)\">\n",
162+
"<g clip-path=\"url(#clip3300)\">\n",
162163
"<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:end;\" transform=\"rotate(0, 129.898, 902.307)\" x=\"129.898\" y=\"902.307\">0.4</text>\n",
163164
"</g>\n",
164-
"<g clip-path=\"url(#clip3200)\">\n",
165+
"<g clip-path=\"url(#clip3300)\">\n",
165166
"<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:end;\" transform=\"rotate(0, 129.898, 619.92)\" x=\"129.898\" y=\"619.92\">0.6</text>\n",
166167
"</g>\n",
167-
"<g clip-path=\"url(#clip3200)\">\n",
168+
"<g clip-path=\"url(#clip3300)\">\n",
168169
"<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:end;\" transform=\"rotate(0, 129.898, 337.533)\" x=\"129.898\" y=\"337.533\">0.8</text>\n",
169170
"</g>\n",
170-
"<polyline clip-path=\"url(#clip3202)\" style=\"stroke:#009af9; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
171+
"<polyline clip-path=\"url(#clip3302)\" style=\"stroke:#009af9; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
171172
" 216.13,1446.71 237.297,1442.3 258.464,1436.56 279.632,1431.36 300.799,1425.04 321.966,1417.02 343.134,1410.31 364.301,1404.01 385.468,1396.38 406.635,1388.22 \n",
172173
" 427.803,1378.49 448.97,1368.17 470.137,1357.5 491.305,1351.4 512.472,1344.49 533.639,1335.66 554.806,1327.3 575.974,1316.56 597.141,1303.97 618.308,1296.16 \n",
173174
" 639.476,1285.85 660.643,1271.85 681.81,1260.98 702.977,1246.06 724.145,1223.67 745.312,1219.3 766.479,1212.74 787.647,1204.13 808.814,1196.33 829.981,1186.85 \n",
@@ -179,16 +180,16 @@
179180
" 1909.51,472.673 1930.68,457.581 1951.85,443.408 1973.01,426.225 1994.18,407.866 2015.35,385.975 2036.52,362.768 2057.68,338.688 2078.85,325.039 2100.02,309.486 \n",
180181
" 2121.19,289.607 2142.35,270.793 2163.52,246.643 2184.69,218.271 2205.85,200.75 2227.02,177.534 2248.19,146.021 2269.36,121.579 2290.52,88.0053 \n",
181182
" \"/>\n",
182-
"<polygon clip-path=\"url(#clip3200)\" points=\"\n",
183-
"1989.93,251.724 2280.76,251.724 2280.76,130.764 1989.93,130.764 \n",
183+
"<path clip-path=\"url(#clip3300)\" d=\"\n",
184+
"M1989.93 251.724 L2280.76 251.724 L2280.76 130.764 L1989.93 130.764 Z\n",
184185
" \" fill=\"#ffffff\" fill-rule=\"evenodd\" fill-opacity=\"1\"/>\n",
185-
"<polyline clip-path=\"url(#clip3200)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
186+
"<polyline clip-path=\"url(#clip3300)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
186187
" 1989.93,251.724 2280.76,251.724 2280.76,130.764 1989.93,130.764 1989.93,251.724 \n",
187188
" \"/>\n",
188-
"<polyline clip-path=\"url(#clip3200)\" style=\"stroke:#009af9; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
189+
"<polyline clip-path=\"url(#clip3300)\" style=\"stroke:#009af9; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
189190
" 2013.93,191.244 2157.93,191.244 \n",
190191
" \"/>\n",
191-
"<g clip-path=\"url(#clip3200)\">\n",
192+
"<g clip-path=\"url(#clip3300)\">\n",
192193
"<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:start;\" transform=\"rotate(0, 2181.93, 208.744)\" x=\"2181.93\" y=\"208.744\">y1</text>\n",
193194
"</g>\n",
194195
"</svg>\n"
@@ -200,15 +201,15 @@
200201
}
201202
],
202203
"source": [
203-
"V = TabularVApproximator(1+WinCapital)\n",
204+
"V = TabularApproximator(n_state=1+WinCapital)\n",
204205
"value_iteration!(V=V, model=GamblerProblemEnvModel, γ=1.0, max_iter=1000)\n",
205206
"plot(V.table[2:end-1])"
206207
]
207208
}
208209
],
209210
"metadata": {
210211
"kernelspec": {
211-
"display_name": "Julia 1.3.0-rc2",
212+
"display_name": "Julia 1.3.0",
212213
"language": "julia",
213214
"name": "julia-1.3"
214215
},

0 commit comments

Comments
 (0)