0x6 Python教程：Spidering

#!/usr/bin/pythonfrom spider import webspider as myspiderimport sys, optparse def crawler(URLs):        for line in open(URLs, 'r'):                URL = line.strip()                links = myspider(b=URL.strip(), w=200, d=5, t=5)                link_count = len(links[0])                out = URL+": has a link count of "+str(link_count)                print "[+] Web Crawl Results for: "+URL                print out                for item in links[1]:                        print item def main():# This optparse module allows you to build command line switches in your scripts# This will set the usage to '-r' and have it stored to a variable URLs# Then we will open the file given at the command line with -r and attempt to spider        parser = optparse.OptionParser(sys.argv[0]+' '+        '-r <file_with URLs>')        parser.add_option('-r', dest='URLs', type='string',                 help='specify target file with URLs')        (options, args) = parser.parse_args()        URLs=options.URLs         if (URLs == None):                print parser.usage                sys.exit(0)        else:                crawler(URLs) if __name__ == "__main__":      main()