Add initial tests

Author: mpskovvang

.gitignore

@@ -1 +1,9 @@
+# Binaries and build artifacts
+*.pyc
+*.o
+*.so
+build/
+dist/
+robotstextparser.egg-info/
+
 /.idea

Dockerfile

@@ -0,0 +1,8 @@
+FROM python:3.7.1
+
+WORKDIR /usr/src/app
+#ENV PYTHONPATH /usr/src/app
+#COPY requirements.txt ./
+#RUN pip install --no-cache-dir -r requirements.txt
+
+COPY . .

dev-requirements.txt

@@ -0,0 +1,2 @@
+nose==1.3.7
+rednose==1.1.1

docker-compose.yml

@@ -0,0 +1,6 @@
+version: '3.7'
+services:
+  python:
+    build: .
+    volumes:
+    - .:/usr/src/app

setup.cfg

@@ -0,0 +1,5 @@
+[nosetests]
+verbosity=2
+rednose=1
+exe=1
+logging-clear-handlers=1

tests/__init__.py

@@ -0,0 +1,165 @@
+import codecs
+import unittest
+
+from robotstxtparser import robotstxtparser
+
+
+class RobotsTest(unittest.TestCase):
+
+    def test_honors_specific_agent(self):
+        """Honors the specific user agent if a match is found"""
+        rp = robotstxtparser.RobotExclusionRulesParser()
+        rp.parse('''
+        User-agent: *
+        Disallow: /tmp
+        
+        User-agent: agent
+        Allow: /tmp
+        ''')
+
+        self.assertTrue(rp.is_allowed('agent', 'http://example.org/tmp'))
+        self.assertTrue(rp.is_allowed('agent', 'http://example.org/path'))
+
+    def test_grouping(self):
+        """Multiple consecutive User-Agent lines are allowed."""
+        rp = robotstxtparser.RobotExclusionRulesParser()
+        rp.parse('''
+            User-agent: one
+            User-agent: two
+            Disallow: /tmp
+        ''')
+
+        self.assertFalse(rp.is_allowed('one', 'http://example.org/tmp'))
+        self.assertFalse(rp.is_allowed('two', 'http://example.org/tmp'))
+        self.assertTrue(rp.is_allowed('agent', 'http://example.org/tmp'))
+
+    def test_grouping_unknown_keys(self):
+        """
+        When we encounter unknown keys, we should disregard any grouping that may have
+        happened between user agent rules.
+
+        This is an example from the wild. Despite `Noindex` not being a valid directive,
+        we'll not consider the '*' and 'ia_archiver' rules together.
+        """
+        rp = robotstxtparser.RobotExclusionRulesParser()
+        rp.parse('''
+            User-agent: *
+            Disallow: /content/2/
+            User-agent: *
+            Noindex: /gb.html
+            Noindex: /content/2/
+            User-agent: ia_archiver
+            Disallow: /
+        ''')
+
+        self.assertTrue(rp.is_allowed('agent', 'http://example.org/foo'))
+        self.assertTrue(rp.is_allowed('ia_archiver', 'http://example.org/bar'))
+
+    def test_case_insensitivity(self):
+        """Make sure user agent matches are case insensitive"""
+        rp = robotstxtparser.RobotExclusionRulesParser()
+        rp.parse('''
+            User-agent: Agent
+            Disallow: /path
+        ''')
+
+        self.assertFalse(rp.is_allowed('agent', 'http://example.org/path'))
+        self.assertFalse(rp.is_allowed('aGeNt', 'http://example.org/path'))
+
+    def test_skip_malformed_line(self):
+        """If there is no colon in a line, then we must skip it"""
+        rp = robotstxtparser.RobotExclusionRulesParser()
+        rp.parse('''
+            User-Agent: agent
+            Disallow /no/colon/in/this/line
+        ''')
+
+        self.assertTrue(rp.is_allowed('agent', 'http://example.org/no/colon/in/this/line'))
+
+    def test_utf8_bom(self):
+        """If there's a utf-8 BOM, we should parse it as such"""
+        rp = robotstxtparser.RobotExclusionRulesParser()
+        rp.parse(codecs.BOM_UTF8 + b'''
+            User-Agent: agent
+            Allow: /path
+            User-Agent: other
+            Disallow: /path
+        ''')
+
+        self.assertTrue(rp.is_allowed('agent', 'http://example.org/path'))
+        self.assertFalse(rp.is_allowed('other', 'http://example.org/path'))
+
+    def test_rfc_example(self):
+        """Tests the example provided by the RFC"""
+        rp = robotstxtparser.RobotExclusionRulesParser()
+        rp.parse('''
+            # /robots.txt for http://www.fict.org/
+            # comments to webmaster@fict.org
+            
+            User-agent: unhipbot
+            Disallow: /
+            
+            User-agent: webcrawler
+            User-agent: excite
+            Disallow:
+            
+            User-agent: *
+            Disallow: /org/plans.html
+            Allow: /org/
+            Allow: /serv
+            Allow: /~mak
+            Disallow: /
+        ''')
+
+        # The unhipbot bot
+        self.assertFalse(rp.is_allowed('unhipbot', 'http://example.org/'))
+        self.assertFalse(rp.is_allowed('unhipbot', 'http://example.org/index.html'))
+        #self.assertTrue(rp.is_allowed('unhipbot', 'http://example.org/robots.txt')) TODO
+        self.assertFalse(rp.is_allowed('unhipbot', 'http://example.org/server.html'))
+        self.assertFalse(rp.is_allowed('unhipbot', 'http://example.org/services/fast.html'))
+        self.assertFalse(rp.is_allowed('unhipbot', 'http://example.org/services/slow.html'))
+        self.assertFalse(rp.is_allowed('unhipbot', 'http://example.org/orgo.gif'))
+        self.assertFalse(rp.is_allowed('unhipbot', 'http://example.org/org/about.html'))
+        self.assertFalse(rp.is_allowed('unhipbot', 'http://example.org/org/plans.html'))
+        self.assertFalse(rp.is_allowed('unhipbot', 'http://example.org/%7Ejim/jim.html'))
+        self.assertFalse(rp.is_allowed('unhipbot', 'http://example.org/%7Emak/mak.html'))
+
+        # The webcrawler bot
+        self.assertTrue(rp.is_allowed('webcrawler', 'http://example.org/'))
+        self.assertTrue(rp.is_allowed('webcrawler', 'http://example.org/index.html'))
+        self.assertTrue(rp.is_allowed('webcrawler', 'http://example.org/robots.txt'))
+        self.assertTrue(rp.is_allowed('webcrawler', 'http://example.org/server.html'))
+        self.assertTrue(rp.is_allowed('webcrawler', 'http://example.org/services/fast.html'))
+        self.assertTrue(rp.is_allowed('webcrawler', 'http://example.org/services/slow.html'))
+        self.assertTrue(rp.is_allowed('webcrawler', 'http://example.org/orgo.gif'))
+        self.assertTrue(rp.is_allowed('webcrawler', 'http://example.org/org/about.html'))
+        self.assertTrue(rp.is_allowed('webcrawler', 'http://example.org/org/plans.html'))
+        self.assertTrue(rp.is_allowed('webcrawler', 'http://example.org/%7Ejim/jim.html'))
+        self.assertTrue(rp.is_allowed('webcrawler', 'http://example.org/%7Emak/mak.html'))
+
+        # The excite bot
+        self.assertTrue(rp.is_allowed('excite', 'http://example.org/'))
+        self.assertTrue(rp.is_allowed('excite', 'http://example.org/index.html'))
+        self.assertTrue(rp.is_allowed('excite', 'http://example.org/robots.txt'))
+        self.assertTrue(rp.is_allowed('excite', 'http://example.org/server.html'))
+        self.assertTrue(rp.is_allowed('excite', 'http://example.org/services/fast.html'))
+        self.assertTrue(rp.is_allowed('excite', 'http://example.org/services/slow.html'))
+        self.assertTrue(rp.is_allowed('excite', 'http://example.org/orgo.gif'))
+        self.assertTrue(rp.is_allowed('excite', 'http://example.org/org/about.html'))
+        self.assertTrue(rp.is_allowed('excite', 'http://example.org/org/plans.html'))
+        self.assertTrue(rp.is_allowed('excite', 'http://example.org/%7Ejim/jim.html'))
+        self.assertTrue(rp.is_allowed('excite', 'http://example.org/%7Emak/mak.html'))
+
+        # All others
+        self.assertFalse(rp.is_allowed('anything', 'http://example.org/'))
+        self.assertFalse(rp.is_allowed('anything', 'http://example.org/index.html'))
+        #self.assertTrue(rp.is_allowed('anything', 'http://example.org/robots.txt')) TODO
+        self.assertTrue(rp.is_allowed('anything', 'http://example.org/server.html'))
+        self.assertTrue(rp.is_allowed('anything', 'http://example.org/services/fast.html'))
+        self.assertTrue(rp.is_allowed('anything', 'http://example.org/services/slow.html'))
+        self.assertFalse(rp.is_allowed('anything', 'http://example.org/orgo.gif'))
+        self.assertTrue(rp.is_allowed('anything', 'http://example.org/org/about.html'))
+        self.assertFalse(rp.is_allowed('anything', 'http://example.org/org/plans.html'))
+        self.assertFalse(rp.is_allowed('anything', 'http://example.org/%7Ejim/jim.html'))
+        self.assertTrue(rp.is_allowed('anything', 'http://example.org/%7Emak/mak.html'))
+