Python爬取视频的示例分析-创新互联
这篇文章给大家分享的是有关Python爬取视频的示例分析的内容。小编觉得挺实用的，因此分享给大家做个参考，一起跟随小编过来看看吧。
先找一个视频地址试验一下：
# -*- coding: utf-8 -*- import requests def download_file(url, path): with requests.get(url, stream=True) as r: chunk_size = 1024 content_size = int(r.headers['content-length']) print '下载开始' with open(path, "wb") as f: for chunk in r.iter_content(chunk_size=chunk_size): f.write(chunk) if __name__ == '__main__': url = '就在原帖...' path = '想存哪都行' download_file(url, path)
遭遇当头一棒：
AttributeError: __exit__
这文档也会骗人的么！
看样子是没有实现上下文需要的__exit__方法。既然只是为了保证要让r最后close以释放连接池，那就使用contextlib的closing特性好了：
# -*- coding: utf-8 -*- import requests from contextlib import closing def download_file(url, path): with closing(requests.get(url, stream=True)) as r: chunk_size = 1024 content_size = int(r.headers['content-length']) print '下载开始' with open(path, "wb") as f: for chunk in r.iter_content(chunk_size=chunk_size): f.write(chunk)
程序正常运行了，不过我盯着这文件，怎么大小不见变啊，到底是完成了多少了呢？还是要让下好的内容及时存进硬盘，还能省点内存是不是：
# -*- coding: utf-8 -*- import requests from contextlib import closing import os def download_file(url, path): with closing(requests.get(url, stream=True)) as r: chunk_size = 1024 content_size = int(r.headers['content-length']) print '下载开始' with open(path, "wb") as f: for chunk in r.iter_content(chunk_size=chunk_size): f.write(chunk) f.flush() os.fsync(f.fileno())
文件以肉眼可见的速度在增大，真心疼我的硬盘，还是最后一次写入硬盘吧，程序中记个数就好了：
def download_file(url, path):
with closing(requests.get(url, stream=True)) as r:
chunk_size = 1024
content_size = int(r.headers['content-length'])
print '下载开始'
with open(path, "wb") as f:
n = 1
for chunk in r.iter_content(chunk_size=chunk_size):
loaded = n*1024.0/content_size
f.write(chunk)
print '已下载{0:%}'.format(loaded)
n += 1结果就很直观了：
已下载2.579129% 已下载2.581255% 已下载2.583382% 已下载2.585508%
心怀远大理想的我怎么会只满足于这一个呢，写个类一起使用吧：
# -*- coding: utf-8 -*-
import requests
from contextlib import closing
import time
def download_file(url, path):
with closing(requests.get(url, stream=True)) as r:
chunk_size = 1024*10
content_size = int(r.headers['content-length'])
print '下载开始'
with open(path, "wb") as f:
p = ProgressData(size = content_size, unit='Kb', block=chunk_size)
for chunk in r.iter_content(chunk_size=chunk_size):
f.write(chunk)
p.output()
class ProgressData(object):
def __init__(self, block,size, unit, file_name='', ):
self.file_name = file_name
self.block = block/1000.0
self.size = size/1000.0
self.unit = unit
self.count = 0
self.start = time.time()
def output(self):
self.end = time.time()
self.count += 1
speed = self.block/(self.end-self.start) if (self.end-self.start)>0 else 0
self.start = time.time()
loaded = self.count*self.block
progress = round(loaded/self.size, 4)
if loaded >= self.size:
print u'%s下载完成\r\n'%self.file_name
else:
print u'{0}下载进度{1:.2f}{2}/{3:.2f}{4} 下载速度{5:.2%} {6:.2f}{7}/s'.\
format(self.file_name, loaded, self.unit,\
self.size, self.unit, progress, speed, self.unit)
print '%50s'%('/'*int((1-progress)*50))运行：
下载开始 下载进度10.24Kb/120174.05Kb 0.01% 下载速度4.75Kb/s ///////////////////////////////////////////////// 下载进度20.48Kb/120174.05Kb 0.02% 下载速度32.93Kb/s /////////////////////////////////////////////////
看上去舒服多了。
下面要做的就是多线程同时下载了，主线程生产url放入队列，下载线程获取url：
# -*- coding: utf-8 -*-
import requests
from contextlib import closing
import time
import Queue
import hashlib
import threading
import os
def download_file(url, path):
with closing(requests.get(url, stream=True)) as r:
chunk_size = 1024*10
content_size = int(r.headers['content-length'])
if os.path.exists(path) and os.path.getsize(path)>=content_size:
print '已下载'
return
print '下载开始'
with open(path, "wb") as f:
p = ProgressData(size = content_size, unit='Kb', block=chunk_size, file_name=path)
for chunk in r.iter_content(chunk_size=chunk_size):
f.write(chunk)
p.output()
class ProgressData(object):
def __init__(self, block,size, unit, file_name='', ):
self.file_name = file_name
self.block = block/1000.0
self.size = size/1000.0
self.unit = unit
self.count = 0
self.start = time.time()
def output(self):
self.end = time.time()
self.count += 1
speed = self.block/(self.end-self.start) if (self.end-self.start)>0 else 0
self.start = time.time()
loaded = self.count*self.block
progress = round(loaded/self.size, 4)
if loaded >= self.size:
print u'%s下载完成\r\n'%self.file_name
else:
print u'{0}下载进度{1:.2f}{2}/{3:.2f}{4} {5:.2%} 下载速度{6:.2f}{7}/s'.\
format(self.file_name, loaded, self.unit,\
self.size, self.unit, progress, speed, self.unit)
print '%50s'%('/'*int((1-progress)*50))
queue = Queue.Queue()
def run():
while True:
url = queue.get(timeout=100)
if url is None:
print u'全下完啦'
break
h = hashlib.md5()
h.update(url)
name = h.hexdigest()
path = 'e:/download/' + name + '.mp4'
download_file(url, path)
def get_url():
queue.put(None)
if __name__ == '__main__':
get_url()
for i in xrange(4):
t = threading.Thread(target=run)
t.daemon = True
t.start()感谢各位的阅读！关于“Python爬取视频的示例分析”这篇文章就分享到这里了，希望以上内容可以对大家有一定的帮助，让大家可以学到更多知识，如果觉得文章不错，可以把它分享出去让更多的人看到吧！
另外有需要云服务器可以了解下创新互联scvps.cn，海内外云服务器15元起步，三天无理由+7*72小时售后在线，公司持有idc许可证，提供“云服务器、裸金属服务器、高防服务器、香港服务器、美国服务器、虚拟主机、免备案服务器”等云主机租用服务以及企业上云的综合解决方案，具有“安全稳定、简单易用、服务可用性高、性价比高”等特点与优势，专为企业上云打造定制，能够满足用户丰富、多元化的应用场景需求。
网站题目：Python爬取视频的示例分析-创新互联
网站链接：http://shouzuofang.com/article/hejeh.html