{"name": "John Doe", "age": 18, "address": {"country" : "china", "zip-code": "10000"}}

1 urllib.request:用來(lái)打開(kāi)和讀取URL。
2 urllib.error:包含了urllib.request產(chǎn)生的異常。
3 urllib.parse:用來(lái)解析和處理URL。
4 urllib.robotparse:用于解析robots.txt文件。

1 urllib.urlopen(url[, data[, proxies[,timeout[, context]]]]) 
功能說(shuō)明:urllib是用于訪問(wèn)URL的唯一方法  
參數(shù)解釋：
      url ：一個(gè)完整的遠(yuǎn)程資源路徑，一般是一個(gè)網(wǎng)站。
      data:默認(rèn)值為none。若參數(shù)data為none，則代表請(qǐng)求方式為get，反之請(qǐng)求方式為post，發(fā)送post請(qǐng)求。參數(shù)data以字典形式存儲(chǔ)數(shù)據(jù)，并將參數(shù)data由字典類型轉(zhuǎn)換成字節(jié)類型才能完成post請(qǐng)求。
      proxies ： 設(shè)置代理
      timeout:超時(shí)設(shè)置
      context:描述各種SSL選項(xiàng)的實(shí)例。
2  urllib.request.Request(url,data=none,headers={},method=none)

功能說(shuō)明:聲明一個(gè)request對(duì)象，該對(duì)象可定義header等請(qǐng)求信息
參數(shù)解釋：
      headers：設(shè)置request請(qǐng)求頭信息
      method：設(shè)定請(qǐng)求方式，主要是get和post

"""urllib的使用"""
import urllib.request

# 向指定URL發(fā)送請(qǐng)求，獲取響應(yīng)。
response = urllib.request.urlopen('http:///anything')
# 獲取響應(yīng)內(nèi)容
content = response.read().decode('utf-8')
print(content)
print(type(response))
# 響應(yīng)碼
print(response.status)
# 響應(yīng)頭信息
print(response.headers)

# 導(dǎo)入urllib
import urllib.request
url = 'https://movie.douban.com/'
# 自定義請(qǐng)求頭
headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko)'
    'Chrome/45.0.2454.85 Safari/537.36 115Browser/6.0.3',
    'Referer': 'https://movie.douban.com/',
    'Connection': 'keep-alive'}
# 設(shè)置request的請(qǐng)求頭
req = urllib.request.Request(url, headers=headers)
# 使用urlopen打開(kāi)req
html = urllib.request.urlopen(req).read().decode('utf-8')
# 寫入文件
f = open('code2.txt', 'w', encoding='utf8')
f.write(html)
f.close()

1 requests.get(url, params=none, **kwargs)
2 requests.post(url, data=none, json=none, **kwargs)

 1 r.status_code：響應(yīng)狀態(tài)碼（200表示訪問(wèn)成功，4**表示失?。? 2 r.raw：原始響應(yīng)體，使用r.raw.read()讀取
 3 r.content：字節(jié)方式的響應(yīng)體，需要進(jìn)行解碼
 4 r.text：字符串方式的響應(yīng)體，會(huì)自動(dòng)根據(jù)響應(yīng)頭部的字符編碼進(jìn)行解碼
 5 r.headers：以字典對(duì)象存儲(chǔ)服務(wù)器響應(yīng)頭，若鍵不存在，則返回none
 6 r.json()：requests中內(nèi)置的json解碼器
 7 r.raise_for_status()：請(qǐng)求失?。ǚ?00響應(yīng)），拋出異常
 8 r.url：獲取請(qǐng)求鏈接
 9 r.cookies：獲取請(qǐng)求后的cookies
10 r.encoding：獲取編碼格式

"""使用requests發(fā)送請(qǐng)求和攜帶參數(shù)"""

import requests

r = requests.get('https:///get')  # 發(fā)送get請(qǐng)求
print(r.text)

# 發(fā)送post請(qǐng)求，并帶參數(shù)
r = requests.get('https:///get', params={'key1': 'value1', 'key2': 'value2'})
print(r.text)

# 發(fā)送post請(qǐng)求，并傳遞參數(shù)
r = requests.post('https:///post', data={'key': 'value'})
print(r.text)

# 其他HTTP請(qǐng)求類型：PUT，DELETE，HEAD和OPTIONS
r = requests.put('https:///put', data={'key': 'value'})
print(r.text)

r = requests.delete('https:///delete')
print(r.text)

r = requests.head('https:///get')
print(r.text)

r = requests.options('https:///get')
print(r.text)

（1）添加請(qǐng)求頭
import requests
headers = {
    'content-type': 'application/json',
    'User-Agent': 'Mozilla/5.0 (Windows NT 6.3; WOW64; rv:41.0) Gecko/20100101 Firefox/41.0'}
requests.get("https://www.baidu.com/", headers=headers)

（2）使用代理IP
import requests
proxies = {
  "http": "http://10.10.1.10:3128",
  "https": "http://10.10.1.10:1080",
}
requests.get("https://www.baidu.com/", proxies=proxies)
（3）證書(shū)驗(yàn)證
import requests
url = 'https://kyfw.12306.cn/otn/leftTicket/init'
# 關(guān)閉證書(shū)驗(yàn)證
r = requests.get(url, verify=False)
print(r.status_code)
# 開(kāi)啟證書(shū)驗(yàn)證
# r = requests.get(url, verify=True)
# 設(shè)置證書(shū)所在路徑
# r = requests.get(url, verify= '/path/to/certfile')

（4）超時(shí)設(shè)置
requests.get("https://www.baidu.com/", timeout=2)
requests.post("https://www.baidu.com/", timeout=2)

(5）使用cookies

import requests
temp_cookies='JSESSIONID_GDS=y4p7osFr_IYV5Udyd6c1drWE8MeTpQn0Y58Tg8cCONVP020y2N!450649273;name=value'
cookies_dict = {}
for i in temp_cookies.split(';'):
    value = i.split('=')
    cookies_dict [value[0]] = value[1]
r = requests.get(url, cookies=cookies)
print(r.text)

"""match方法"""

import re

# match在起始位置匹配
ret = re.match('www', 'www.')
print(type(ret))
# 獲取匹配的內(nèi)容
print(ret.group())
# 獲取匹配內(nèi)容在原字符串里的下標(biāo)
print(ret.span())
# 匹配不成功，返回None
print(re.match('com', 'www.'))

line = "Cats are smarter than dogs"
matchObj = re.match(r'(.*) are (.*?) .*', line, re.M | re.I)
if matchObj:
    print("matchObj.group() : ", matchObj.group())
    # 獲取第一組的內(nèi)容
    print("matchObj.group(1) : ", matchObj.group(1))
    print("matchObj.group(2) : ", matchObj.group(2))
else:
    print("No match!!")

參數(shù)說(shuō)明：
· pattern:匹配的正則表達(dá)式
· string：要匹配的字符串
· flags：標(biāo)志位，用于控制正則表達(dá)式的匹配方式，如是否區(qū)分大小寫，是否多行匹配等。
flags參數(shù)可選值：
· re.I: 忽略大小寫（括號(hào)內(nèi)是完整寫法，下同）
· re.M: 多行模式，改變'^'和'$'的行為（參見(jiàn)上圖）
· re.S: 點(diǎn)任意匹配模式，改變'.'的行為
· re.L: 使預(yù)定字符類 \w \W \b \B \s \S 取決于當(dāng)前區(qū)域設(shè)定
· re.U: 使預(yù)定字符類 \w \W \b \B \s \S \d \D 取決于unicode定義的字符屬性
· re.X: 詳細(xì)模式。這個(gè)模式下正則表達(dá)式可以是多行，忽略空白字符，并可以加入注釋。

"""search方法"""

import re

# search查找第一次出現(xiàn)
ret = re.search('www', 'www.aaa.com www.bbb.com')
print(type(ret))
print(ret.group())
print(ret.span())
#匹配不成功，返回None
print(re.search('cn', 'www.aaa.com www.bbb.com'))


line = "Cats are smarter than dogs";
searchObj = re.search(r'(.*) are (.*?) .*', line, re.M | re.I)
if searchObj:
    print("searchObj.group() : ", searchObj.group())
    print("searchObj.group(1) : ", searchObj.group(1))
    print("searchObj.group(2) : ", searchObj.group(2))
else:
    print("Nothing found!!")

"""findall方法"""

import re

# 查找數(shù)字
result1 = re.findall(r'\d+','baidu 123 google 456')
result2 = re.findall(r'\d+','baidu88oob123google456')

print(result1)
print(result2)

"""finditer方法"""

import re

#返回一個(gè)迭代器，可以循環(huán)訪問(wèn)，每次獲取一個(gè)Match對(duì)象
it = re.finditer(r"\d+", "12a32bc43jf3")
for match in it:
    print(match.group())

"""sub方法"""

import re

phone = "2004-959-559 # 這是一個(gè)國(guó)外電話號(hào)碼"

# 刪除字符串中的 Python注釋
num = re.sub(r'#.*$', "", phone)
print("電話號(hào)碼是: ", num)

# 刪除非數(shù)字(-)的字符串
num = re.sub(r'\D', "", phone)
print("電話號(hào)碼是 : ", num)


# 將匹配的數(shù)字乘以 2
def double(matched):
    value = int(matched.group('value'))
    return str(value * 2)


s = 'A23G4HFD567'
print(re.sub('(?P<value>\d+)', double, s))

參數(shù)說(shuō)明:
1 repl:用于替換的字符串
2 string:要被替換的字符串
3 count:替換的次數(shù)

"""compile方法"""


import re

# 用于匹配至少一個(gè)數(shù)字
pattern = re.compile(r'\d+')
# 查找頭部，沒(méi)有匹配
m = pattern.match('one12twothree34four')
print(m)
# 從'e'的位置開(kāi)始匹配，沒(méi)有匹配
m = pattern.match('one12twothree34four', 2, 10)
print(m)
# 從'1'的位置開(kāi)始匹配，正好匹配，返回一個(gè) Match 對(duì)象
m = pattern.match('one12twothree34four', 3, 10)
print(m)
# 可省略 0
print(m.group(0))

1 pip install beautifulsoup4

1 from bs4 import BeautifulSoup

"""將字符串解析為HTML文檔解析"""

from bs4 import BeautifulSoup

html = """
<html><head><title>The Dormouse's story</title></head>
<body>
<p class="title" name="dromouse"><b>The Dormouse's story</b></p>
<p class="story">Once upon a time there were three little sisters; and their names were
<a href="http:///elsie" class="sister" id="link1"><!-- Elsie --></a>,
<a href="http:///lacie" class="sister" id="link2">Lacie</a> and
<a href="http:///tillie" class="sister" id="link3">Tillie</a>;
and they lived at the bottom of a well.</p>
<p class="story">...</p>
"""

# 創(chuàng)建BeautifulSoup對(duì)象解析html，并使用lxml作為xml解析器soup = BeautifulSoup(html, 'lxml')

# 格式化輸出soup對(duì)象的內(nèi)容
print(soup.prettify())

例一：
"""bs4實(shí)例測(cè)試"""

from bs4 import BeautifulSoup
import re

html = """
<html><head><title>The Dormouse's story</title></head>
<body>
<p class="title" name="dromouse"><b>The Dormouse's story</b></p>
<p class="story">Once upon a time there were three little sisters; and their names were
<a href="http:///elsie" class="sister" id="link1"><!-- Elsie --></a>,
<a href="http:///lacie" class="sister" id="link2">Lacie</a> and
<a href="http:///tillie" class="sister" id="link3">Tillie</a>;
and they lived at the bottom of a well.</p>
<p class="story">...</p>
"""

# 創(chuàng)建對(duì)象
soup = BeautifulSoup(html, 'lxml')

# 獲取Tag對(duì)象
# 查找的是在所有內(nèi)容中的第一個(gè)符合要求的標(biāo)簽
print(soup.title)
print(type(soup.title))
print(soup.p)
# 標(biāo)簽的名字
print(soup.p.name)
# 標(biāo)簽的屬性，可獲取，也可以設(shè)置
print(soup.p.attrs)
print(soup.p.attrs['class'])

# 文本
print(soup.p.string)

# content屬性得到子節(jié)點(diǎn)，列表類型
print(soup.body.contents)
# children屬性屬性得到子節(jié)點(diǎn)，可迭代對(duì)象
print(soup.body.children)
# descendants屬性屬性得到子孫節(jié)點(diǎn)，可迭代對(duì)象
print(soup.body.descendants)

# find_all 查找所有符合要求的  name是按照標(biāo)簽名字查找
print(soup.find_all(name='b'))
print(soup.find_all(name=['a', 'b']))
print(soup.find_all(name=re.compile("^b")))

# find_all 查找所有符合要求的  可以按照屬性查找，比如這里的id,class
# 因?yàn)閏lass是關(guān)鍵字，所以使用class_代替class
print(soup.find_all(id='link2'))
print(soup.find_all(class_='sister'))

# select 查找所有符合要求的 支持選擇器
print(soup.select('title'))
print(soup.select('.sister'))
print(soup.select('#link1'))
print(soup.select('p #link1'))
print(soup.select('a[class="sister"]'))

# 獲取內(nèi)容
print(soup.select('title')[0].get_text())

例二：

>>> from bs4 import BeautifulSoup
>>> import requests
>>> r = requests.get("http:///ws/demo.html")
>>> demo = r.text
>>> demo
'<html><head><title>This is a python demo page</title></head>\r\n<body>\r\n<p class="title"><b>The demo python introduces several python courses.</b></p>\r\n<p class="course">Python is a wonderful general-purpose programming language. You can learn Python from novice to professional by tracking the following courses:\r\n<a  class="py1" id="link1">Basic Python</a> and <a  class="py2" id="link2">Advanced Python</a>.</p>\r\n</body></html>'
>>> soup = BeautifulSoup(demo,"html.parser")
>>> soup.title #獲取標(biāo)題
<title>This is a python demo page</title>
>>> soup.a #獲取a標(biāo)簽
<a class="py1"  id="link1">Basic Python</a>
>>> soup.title.string
'This is a python demo page'
>>> soup.prettify() #輸出html標(biāo)準(zhǔn)格式內(nèi)容
'<html>\n <head>\n <title>\n This is a python demo page\n </title>\n </head>\n <body>\n <p class="title">\n <b>\n The demo python introduces several python courses.\n </b>\n </p>\n <p class="course">\n Python is a wonderful general-purpose programming language. You can learn Python from novice to professional by tracking the following courses:\n <a class="py1"  id="link1">\n Basic Python\n </a>\n and\n <a class="py2"  id="link2">\n Advanced Python\n </a>\n .\n </p>\n </body>\n</html>'
>>> soup.a.name #每個(gè)<tag>都有自己的名字，通過(guò)<tag>.name獲取
'a'
>>> soup.p.name
'p'
>>> tag = soup.a
>>> tag.attrs
{'href': 'http://www.icourse163.org/course/BIT-268001', 'class': ['py1'], 'id': 'link1'}
>>> tag.attrs['class']
['py1']
>>> tag.attrs['href']
'http://www.icourse163.org/course/BIT-268001'
>>> type(tag.attrs)
<class 'dict'>
>>> type(tag)
<class 'bs4.element.Tag'>

# lxml安裝：
pip install lxml
# 使用
（1）
"""將字符串解析為HTML文檔"""

from lxml import etree

text='''
<div>
<ul>
<liclass="item-0"><ahref="link1.html">firstitem</a></li>
<liclass="item-1"><ahref="link2.html">seconditem</a></li>
<liclass="item-inactive"><ahref="link3.html">thirditem</a></li>
<liclass="item-1"><ahref="link4.html">fourthitem</a></li>
<liclass="item-0"><ahref="link5.html">fifthitem</a>
</ul>
</div>
'''

#利用etree.HTML，將字符串解析為HTML文檔
html=etree.HTML(text)

#按字符串序列化HTML文檔
result=etree.tostring(html).decode('utf-8')

print(result)

（2）
"""讀文件"""

from lxml import etree

# 讀取外部文件hello.html
html = etree.parse('./data/hello.html')
# pretty_print=True表示格式化，比如左對(duì)齊和換行
result = etree.tostring(html, pretty_print=True).decode('utf-8')

print(result)