#!/usr/bin/env python

 from sys import argv

 from os import makedirs, unlink, sep

 from os.path import isdir, exists, dirname, splitext

 from string import replace, find, lower

 from htmllib import HTMLParser

 from urllib import urlretrieve

 from urlparse import urlparse, urljoin

 from formatter import DumbWriter, AbstractFormatter

 from cStringIO import StringIO

 class Retriever(object):    # download Web pages

     def __init__(self, url):

         self.url = url

         self.file = self.filename(url)

     def filename(self, url, deffile='index.htm'):

         parsedurl = urlparse(url, 'http:', 0)  # parse path

         path = parsedurl[1] + parsedurl[2]

         ext = splitext(path)

         if ext[1] == '':

             if path[-1] == '/':

                 path += deffile

             else:

                 path += '/' + deffile

         ldir = dirname(path)    # local directory

     if sep != '/':        # os-indep. path separator

         ldir = replace(ldir, ',', sep)

         if not isdir(ldir):      # create archive dir if nec.

             if exists(ldir): unlink(ldir)

             makedirs(ldir)

         return path

     def download(self):        # download Web page

         try:

             retval = urllib.urlretrieve(self.url, self.file)

         except IOError:

             retval = ('*** ERROR: invalid URL "%s"' % \

                 self.url, )

         return retval

     def parseAndGetLinks(self):    # pars HTML, save links

         self.parser = HTMLParser(AbstractFormatter( \

             DumbWriter(StringIO())))

         self.parser.feed(open(self.file).read())

         self.parser.close()

         return self.parse.anchorlist

 class Crawler(object):        # manage entire crawling process

     count = 0            # static downloaded page counter

     def __init__(self, url):

         self.q = [url]

         self.seen = []

         self.dom = urlparse(url)[1]

     def getPage(self, url):

         r = Retriever(url)

         retval = r.download()

         if retval[0] == '*':     # error situation, do not parse

             print retval, '... skipping parse'

             return

         Crawler.count = Crawler.count + 1

         print '\n(', Crawler.count, ')'

         print 'URL:', url

         print 'FILE:', retval[0]

         self.seen.append(url)

         links = r.parseAndGetLinks()  # get and process links

         for eachLink in links:

             if eachLink[:4] != 'http' and \

                     find(eachLink, '://') == -1:

                 eachLink = urljoin(url, eachLink)

             print '* ', eachLink,

             if find(lower(eachLink), 'mailto:') != -1:

                 print '... discarded, mailto link'

                 continue

             if eachLink not in self.seen:

                 if find(eachLink, self.dom) == -1:

                     print '... discarded, not in domain'

                 else:

                     if eachLink not in self.q:

                         self.q.append(eachLink)

                         print '... new, added to Q'

                     else:

                         print '... discarded, already in Q'

             else:

                     print '... discarded, already processed'

     def go(self):                # process links in queue

         while self.q:

             url = self.q.pop()

             self.getPage(url)

 def main():

     if len(argv) > 1:

         url = argv[1]

     else:

         try:

             url = raw_input('Enter starting URL: ')

         except (KeyboardInterrupt, EOFError):

             url = ''

     if not url: return

     robot = Crawler(url)

     robot.go()

 if __name__ == '__main__':

     main()