Question

我正在使用BeautifulSoup，我正在练习获取网站内容。

但是，在输出中从第二个开始重复，每个重复一次。

我试图在for循环中修改代码，但仍会重复。

#coding:utf-8
import lxml
import json
import re
import requests
from bs4 import BeautifulSoup

def the_url(url):
    user_agent = "Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1"
    headers = {"User-Agent":user_agent}
    r = requests.get(url, headers=headers)
    return r.text

def get_text(page_html):
    the_web = BeautifulSoup(page_html, 'html.parser')

    base_url = "https://cn.reuters.com"

    list_div = the_web.find('div', {"id": 'chinaNews'})

    list_li = list_div.find_all('li')

    for t in list_li:
        the_dict = {}
        a = t.find('a')

        excerpt = t.find('div', {"class": 'smalltext'})

        if a:
            the_dict['link'] = base_url + a.get('href')
            the_dict['title'] = a.get_text()
        if excerpt:
            the_dict['excerpt'] = excerpt.get_text()

        result_list.append(the_dict)


def save_to_json(result):
    s = json.dumps(result, indent = 4, ensure_ascii = False)
    # json file
    with open('text.json', 'w', encoding = 'utf-8') as f:
        f.write(s)

def main():
    for i in range(2):
        i = i + 1
        url = 'http://cn.mobile.reuters.com/category/chinaNews?p={}'.format(i)
        page_html = the_url(url)
        get_text(page_html)
    save_to_json(result_list)

if __name__ == '__main__':
    result_list = []
    main()

我想删除输出中的重复项。

Answer 1

您可以检查该值是否已在字典中：

if the_dict and not any(r['link'] == the_dict['link'] for r in result_list):
    # No dict with this link exist in the result_list
    result_list.append(the_dict)

以下是您的get_text方法中的以下检查：

def get_text(page_html):
    the_web = BeautifulSoup(page_html, 'html.parser')

    base_url = "https://cn.reuters.com"

    list_div = the_web.find('div', {"id": 'chinaNews'})

    list_li = list_div.find_all('li')

    for t in list_li:
        the_dict = {}
        a = t.find('a')

        excerpt = t.find('div', {"class": 'smalltext'})

        if a:
            the_dict['link'] = base_url + a.get('href')
            the_dict['title'] = a.get_text()
        if excerpt:
            the_dict['excerpt'] = excerpt.get_text()


        if the_dict and not any(r['link'] == the_dict['link'] for r in result_list):
            result_list.append(the_dict)

从第二个开始复制输出

1 个答案: