Python 爬虫学习

            #
            coding:utf-8
            
#
            author:Blood_Zero
            ‘‘‘
            
    1、获取网页信息
    2、解决编码问题，通过charset库(默认不安装这个库文件)

            ‘‘‘
            import
             urllib

            import
             urllib2

url = "http://192.168.1.135/myself/"
html = urllib.urlopen(url)
content = html.read()
print content
#如果网页中存在其他编码，就会出现乱码
#print content.decode(‘gbk‘).encode(‘utf-8‘)‘‘‘
    简易获取网页信息
‘‘‘#获取当前urlprint"当前URL："+str(html.geturl())

#网页状态码print"当前状态码："+str(html.code)
#print "当前状态码："+str(html.getcode())#网站头信息print"当前头信息：n"+str(html.headers)
#print "当前头信息：n"+str(html.info())#获取网站编码print"当前网站使用编码："+str(html.info().getparam("charset"))

#下载网页源码
urllib.urlretrieve(url,"E:\Python_Code\pyTools\url.txt")


‘‘‘
    模拟浏览器访问网址
‘‘‘#方法一
req=urllib2.Request(url)
# 添加头信息
req.add_header("User-Agent","Mozilla/5.0 (Windows NT 6.2; WOW64; rv:39.0) Gecko/20100101 Firefox/39.0")
req.add_header("Get",url)
req.add_header("Host","192.168.1.135")

new_html = urllib2.urlopen(req)
print new_html.read()
print req.headers.items()

#方法二
myheader={
    "User-Agent":"Mozilla/5.0 (Windows NT 6.2; WOW64; rv:39.0) Gecko/20100101 Firefox/39.0",
    "Host":"192.168.1.135",
    "Get":url
}
req1 = urllib2.Request(url,headers=myheader)
new_html_1 = urllib2.urlopen(req1)
print new_html_1.read()
print req1.headers.items()


‘‘‘
    在网页中查询指定文件
‘‘‘def get_content(url):
    html = urllib.urlopen(url)
    content = html.read()
    html.close()
    return content

def get_file(self):
    #匹配php文件
    regex = r‘a href=(.+?.php)‘
    pat=re.compile(regex)

    file_code = re.findall(pat,self)
    print str(file_code)+"n"

info = get_content("http://192.168.1.135/myself/SQL_Injection/")
get_file(info)