最新下载
热门教程
- 1
- 2
- 3
- 4
- 5
- 6
- 7
- 8
- 9
- 10
Python 爬虫web网页版程序代码
时间:2015-08-30 编辑:简简单单 来源:一聚教程网
一:网页结构分析
二:代码实战
#! /usr/bin/env python2 # encoding=utf-8 #BeautifulSoup需要安装 MySQLdb import sys,os,re,hashlib import urllib import httplib2 from lxml import etree import MySQLdb from BeautifulSoup import BeautifulSoup import urllib2 import re import time reload(sys) from datetime import datetime as dt,timedelta import re h=httplib2.Http(timeout=10) #设置请求http头 模拟伪装 浏览器 headers={ 'User-Agent':'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0)' } #正则匹配a标签 pattern = '(.*?)' #日志记录 log_path='./sporttery' log_file='%s.log' % dt.now().strftime('%Y-%m-%d') if not os.path.exists(log_path): os.makedirs(log_path) log=open('%s/%s' % (log_path,log_file),'w+') #python操作mysql数据库 conn= MySQLdb.connect( host='localhost', port = 3306, user='root', passwd='root', db ='test', ) conn.set_character_set('utf8') cur = conn.cursor() cur.execute('SET NAMES utf8;') cur.execute('SET CHARACTER SET utf8;') cur.execute('SET character_set_connection=utf8;') cur.close() #获取请求链接内容 失败再次执行 def download(url): fails = 0 while True: if fails>5:return None try: res,content = h.request(url,'GET',headers=headers) return content.decode('utf-8','ignore') except: print(u'打开链接失败'+url) fails +=1 #字符串截取方法 def GetMiddleStr(content,startStr,endStr): startIndex = content.index(startStr) if startIndex>=0: startIndex += len(startStr) endIndex = content.index(endStr) return content[startIndex:endIndex] def get_ul(data): mystring=GetMiddleStr(data,' ','') return mystring def test_sporttery(i): url='http://www.xxx.com/video/video_%E8%B6%B3%E7%90%83%E9%AD%94%E6%96%B9_'+str(i)+'.html' print url #http://www.xxx.com/video/video_%E8%B6%B3%E7%90%83%E9%AD%94%E6%96%B9_2.html source=download(url) data=get_ul(source) datas=data.split('
- ') for each in datas: ret=re.findall(r"(?<=href=\").+?(?=\")|(?<=href=\').+?(?=\')" ,each) for urls in ret: detial=download(urls) if detial: detial_content=GetMiddleStr(detial,'createFlashVideo','m3u8').replace(' ', '') if detial_content: end_url_rex=GetMiddleStr(detial_content+".m3u8",'http://','.m3u8')+"m3u8" #最终的url #title sstree = etree.HTML(detial) ssnodes = sstree.xpath('//*[@id="playVideo"]/div[1]/h2') for ssn in ssnodes: name= ssn.text.strip().replace('/h2>', '') #title=GetMiddleStr(detial,'
').replace(' ', '') #简介 introduction=GetMiddleStr(detial,'video-info">','').replace(' ', '') dr = re.compile(r'<[^>]+>',re.S) introductions = dr.sub('',introduction) end_content=introductions.strip().replace('/span>', '') end_time= time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(time.time()+8*60*60)) #end_times=dt.now().strftime('%Y-%m-%d %H:%i:%S') saveDB(urls,end_url_rex,name,end_content,str(i),end_time) def saveDB(current_url,end_url_rex,names,end_content,page,create_time): #添加select update sql = 'INSERT INTO test.mytables(current_url,end_url_rex,`names`,end_content,page,create_time)\ VALUES (%s,%s,%s,%s,%s,%s)' print sql cur = conn.cursor() cur.execute(sql,(current_url,end_url_rex,names,end_content,page,create_time)) cur.close() conn.commit() if __name__ == '__main__': first="http://www.xxx.com/video/video_%E8%B6%B3%E7%90%83%E9%AD%94%E6%96%B9_1.html" url = urllib2.urlopen(first) content = url.read() soup = BeautifulSoup(content) strs=soup.findAll(attrs={"class":"pagination"}) lists=str(strs[0]) listss=re.findall(r'\d+',lists) count=len(listss) list_string = list(set(listss)) str_num= list_string[-1] i = 1 while i <= int(str_num): test_sporttery(i) i += 1
-
上一个: PHP MySQL 预处理语句学习笔记
相关文章
- [招聘]网页设计师和程序员 深圳市 03-19
- 小程序大作用,让你的网页酷到底 05-14
- PHP导出数据超时的优化建议解读 10-31
- PHP之mysql位运算解析 10-31
- Laravel实现登录跳转功能解析 10-31
- php双向队列解读 10-31