def get_user_data(self,start_url):
html = self.session.get(url=start_url,headers=self.headers,cookies=self.cookies).content
selector = etree.fromstring(html,etree.HTMLParser(encoding='utf-8'))
# BEGIN
if selector.xpath('//div[contains(@class,"c") and contains(@id,"M")]/div[2]'):
user_id = selector.xpath('//div[contains(@class,"c") and contains(@id,"M")]/div[1]/a/@href')
img = selector.xpath('//div[contains(@class,"c") and contains(@id,"M")]/div[2]/a/img/@src')
praise_num = selector.xpath('//div[contains(@class,"c") and contains(@id,"M")]/div[2]/a[3]/text()')
transmit_num = selector.xpath('//div[contains(@class,"c") and contains(@id,"M")]/div[2]/a[4]/text()')
else:
user_id = selector.xpath('//div[contains(@class,"c") and contains(@id,"M")]/div/a/@href')
img = ''
praise_num = selector.xpath('//div[contains(@class,"c") and contains(@id,"M")]/div/a[3]/text()')
transmit_num = selector.xpath('//div[contains(@class,"c") and contains(@id,"M")]/div[2]/a[4]')
# OVER
contents = selector.xpath('//span[@class="ctt"]/text()')
times = selector.xpath('//span[@class="ct"]/text()')
for each_text, each_time in zip(contents,times):
data = {}
data['content'] = each_text.encode().decode('utf-8').replace('\u200b','')
try:
if re.search('from',each_time.encode().decode('utf-8')):
month_day, time, device = each_time.split(maxsplit=2)
data['mobile_phone'] = device
else:
month_day, time = each_time.split(maxsplit=1)
data['mobile_phone'] = ''
data['create_time'] = month_day +' '+ time
data['crawl_time'] = datetime.strftime(datetime.now(),'%Y-%m-%d %H:%M:%S')
这是用户上传的HTML
<div class="c" id="M_Fp01sdJgm">
<div>
<a class="nk" href="https://weibo.cn/thebs">figre</a>
<img src="https://h5.sinaimg.cn/upload/2016/05/26/319/5338.gif" alt="V"/>
<img src="https://h5.sinaimg.cn/upload/2016/05/26/319/donate_btn_s.png" alt="M"/>
<span class="ctt">
":"resampling
<span class="kt">resampling</span>
":Cleantech entrepreneurs are splicing genes in the search for greener fuels
</span>
[<a href="https://weibo.cn/mblog/picAll/Fp01sdJgm?rl=2">2 pieces of the package</a>
</div>
<div>
<a href="https://weibo.cn/mblog/pic/Fp01sdJgm?rl=1">
<img src="http://wx1.sinaimg.cn/wap180/3ed2e6e8gy1fk7hohl2i5j219s0ps4qp.jpg" alt="images" class="ib" />
</a>
<a href="https://weibo.cn/mblog/oripic?id=Fp01sdJgm&u=3ed2e6e8gy1fk7hohl2i5j219s0ps4qp">image</a>
<a href="https://weibo.cn/attitude/Fp01sdJgm/add?uid=5757914684&rl=1&st=7b15a6">praise[28094]</a>
<a href="https://weibo.cn/repost/Fp01sdJgm?uid=1054009064&rl=1">transmit[1164]</a>
<a href="https://weibo.cn/comment/Fp01sdJgm?uid=1054009064&rl=1#cmtfrm" class="cc">comment[4097]</a> <a href="https://weibo.cn/fav/addFav/Fp01sdJgm?rl=1&st=7b15a6">save</a>
"<!----> "
<span class="ct">10月05日 20:08 from iPhone 7 Plus
用户未上传图片
<div class="c" id="M_Fp01sdJgm">
<div>
<a class="nk" href="https://weibo.cn/thebs">figre</a>
<a href="https://weibo.cn/mblog/pic/Fp01sdJgm?rl=1">
<img src="http://wx1.sinaimg.cn/wap180/3ed2e6e8gy1fk7hohl2i5j219s0ps4qp.jpg" alt="images" class="ib" />
</a>
<a href="https://weibo.cn/mblog/oripic?id=Fp01sdJgm&u=3ed2e6e8gy1fk7hohl2i5j219s0ps4qp">image</a>
<a href="https://weibo.cn/attitude/Fp01sdJgm/add?uid=5757914684&rl=1&st=7b15a6">praise[28094]</a>
<a href="https://weibo.cn/repost/Fp01sdJgm?uid=1054009064&rl=1">transmit[1164]</a>
<a href="https://weibo.cn/comment/Fp01sdJgm?uid=1054009064&rl=1#cmtfrm" class="cc">comment[4097]</a> <a href="https://weibo.cn/fav/addFav/Fp01sdJgm?rl=1&st=7b15a6">save</a>
"<!----> "
<span class="ct">10月05日 20:08 from iPhone 7 Plus
我尝试用HTML获取数据并通过zip()将它们放在一起。
我相信#BEGIN和#OVER之间的代码是错误的。
我需要确定用户是否已上传图像，然后解析HTML。然后将字符串保存到MySQL。现在我需要得到的是user_id，img，praise_num和transmit_num，然后把它们放在一起。我该怎么写呢？