分享几套常用的Python多线程、进程模板,爬虫效率杠杠的
多线程模板一
# -*- coding: utf-8 -*-
“””
当然在学习Python的道路上肯定会困难,没有好的学习资料,怎么去学习呢?
学习Python中有不明白推荐加入交流Q群号:928946953
群里有志同道合的小伙伴,互帮互助, 群里有不错的视频学习教程和PDF!
还有大牛解答!
“””
import time from queue import Queue from threading import Thread class Demo(Thread): def __init__(self, queue): Thread.__init__(self) self.queue = queue def run(self): while True: q_url = self.queue.get() try: self.parse(q_url) finally: self.queue.task_done() def parse(self, q_url): print("开始解析链接:", q_url) if __name__ == "__main__": start = time.time() base_url = "https://www.baidu.com/{}.html" url_list = [base_url.format(i) for i in range(1, 201)] queue = Queue() for x in range(20): worker = Demo(queue) worker.daemon = True worker.start() for url in url_list: queue.put(url) queue.join() print("下载完毕耗时:{}s".format(round(time.time() - start, 2)))