Always allow /robots.txt

Author: mpskovvang

robotstxtparser/robotstxtparser.py

@@ -251,6 +251,9 @@ def is_url_allowed(self, url, syntax=GYM2008):
 
         url = _unquote_path(url)
 
+        if url == '/robots.txt':
+            return True
+
         done = False
         i = 0
         while not done:

tests/test_robots.py

@@ -114,7 +114,7 @@ def test_rfc_example(self):
         # The unhipbot bot
         self.assertFalse(rp.is_allowed('unhipbot', 'http://example.org/'))
         self.assertFalse(rp.is_allowed('unhipbot', 'http://example.org/index.html'))
-        #self.assertTrue(rp.is_allowed('unhipbot', 'http://example.org/robots.txt')) TODO
+        self.assertTrue(rp.is_allowed('unhipbot', 'http://example.org/robots.txt'))
         self.assertFalse(rp.is_allowed('unhipbot', 'http://example.org/server.html'))
         self.assertFalse(rp.is_allowed('unhipbot', 'http://example.org/services/fast.html'))
         self.assertFalse(rp.is_allowed('unhipbot', 'http://example.org/services/slow.html'))
@@ -153,7 +153,7 @@ def test_rfc_example(self):
         # All others
         self.assertFalse(rp.is_allowed('anything', 'http://example.org/'))
         self.assertFalse(rp.is_allowed('anything', 'http://example.org/index.html'))
-        #self.assertTrue(rp.is_allowed('anything', 'http://example.org/robots.txt')) TODO
+        self.assertTrue(rp.is_allowed('anything', 'http://example.org/robots.txt'))
         self.assertTrue(rp.is_allowed('anything', 'http://example.org/server.html'))
         self.assertTrue(rp.is_allowed('anything', 'http://example.org/services/fast.html'))
         self.assertTrue(rp.is_allowed('anything', 'http://example.org/services/slow.html'))