Question

我正在抓取《米尔班克季刊》上发表的文章。我对有关作者及其机构隶属关系的数据特别感兴趣。我已经使用beautifulsoup和pandas库编写了代码，以便将输出保存为csv。 csv每篇文章包含一行。这意味着对于具有多个作者的文章，“作者”列包含所有作者，而“机构”列包含与文章共同撰写的所有作者机构。相反，我想要的输出是csv每个作者只有一行；换句话说，每篇文章有多行。这是因为我想最终计算每个机构在期刊中的代表人数。

我使用了beautifulsoup .find_all方法来获取我的所有数据。最初，我尝试使用.find_all_next来吸引作者和机构，认为可以容纳多位作者的文章，但对于这些专栏什么也没有返回。

对我来说，重写这段代码以使每个作者都有自己的一行的最佳方式是什么？

import pandas as pd
import numpy as np
import requests
import re
import urllib
from bs4 import BeautifulSoup
from bs4 import SoupStrainer

articletype=list()
articlelist=list()
titlelist=list()
vollist=list()
issuenumlist=list()
authorlist = list()
instlist = list()
urllist=list()

issueurllist = ['https://onlinelibrary.wiley.com/toc/14680009/2018/96/1', 'https://onlinelibrary.wiley.com/toc/14680009/2018/96/2','https://onlinelibrary.wiley.com/toc/14680009/2018/96/3','https://onlinelibrary.wiley.com/toc/14680009/2018/96/4']

for issue in issueurllist:
    requrl = requests.get(issue)
    soup = BeautifulSoup(requrl.text, 'lxml')

    #Open url of each article.

    baseurl = 'https://onlinelibrary.wiley.com'

    for article in issue:
        doi=[a.get('href') for a in soup.find_all('a', title = "Full text")]

    for d in doi:
        doilink = baseurl + d
        opendoi = requests.get(doilink)
        articlesoup=BeautifulSoup(opendoi.text, 'lxml')

    ```Get metadata for each article```
    for tag in articlesoup:
        arttype=articlesoup.find_all("span", {"class":"primary-heading"})
        title=articlesoup.find_all("meta",{"name":"citation_title"})
        vol=articlesoup.find_all("meta",{"name":"citation_volume"})
        issuenum = articlesoup.find_all("meta",{"name":"citation_issue"})
        author = articlesoup.find_all("meta",{"name":"citation_author"})
        institution=articlesoup.find_all("meta",{"name":"citation_author_institution"})
        url=articlesoup.find_all("meta",{"name":"citation_fulltext_html_url"})

    articletype.append(arttype)
    titlelist.append(title)
    vollist.append(vol)
    issuenumlist.append(issuenum)
    authorlist.append(author)
    instlist.append(institution)
    urllist.append(url)

    milbankdict={'article type':articletype, 'title':titlelist, 'vol':vollist, 'issue':issuenumlist,'author':authorlist, 'author institution':instlist, 'url':urllist}
    milbank2018=pd.DataFrame(milbankdict)
    milbank2018.to_csv('milbank2018.csv')
    print("saved")

Answer 1

find_all方法始终返回一个列表，如您所见，我正在验证tag_object is not None，这是一个重要的测试用例，因为某些作者不包含meta属性，然后返回None。不需要每个元属性有多个列表，您可以使用字典进行管理，这里我正在格式化data by author并关联所有元属性。

Python的

strip()内置函数用于删除字符串中的所有前导和尾随空格。

import requests
from bs4 import BeautifulSoup
import pandas as pd

issueurllist = ['https://onlinelibrary.wiley.com/toc/14680009/2018/96/1',
                'https://onlinelibrary.wiley.com/toc/14680009/2018/96/2',
                'https://onlinelibrary.wiley.com/toc/14680009/2018/96/3',
                'https://onlinelibrary.wiley.com/toc/14680009/2018/96/4'
                ]

base_url = 'https://onlinelibrary.wiley.com'

json_data = []

for issue in issueurllist:
    response1 = requests.get(issue)
    soup1 = BeautifulSoup(response1.text, 'lxml')

    for article in issue:
        doi=[a.get('href') for a in soup1.find_all('a', title = "Full text")]

    for i in doi:
        article_dict = {"article":"NaN","title":"NaN","vol":"NaN","issue":"NaN","author":"NaN","institution":"NaN","url":"NaN"}
        article_url = base_url + i
        response2 = requests.get(article_url)
        soup2=BeautifulSoup(response2.text, 'lxml')

        '''Get metadata for each article'''

        article = soup2.find("span", {"class":"primary-heading"})
        title = soup2.find("meta",{"name":"citation_title"})
        vol = soup2.find("meta",{"name":"citation_volume"})
        issue  = soup2.find("meta",{"name":"citation_issue"})
        author  = soup2.find("meta",{"name":"citation_author"})
        institution = soup2.find("meta",{"name":"citation_author_institution"})
        url = soup2.find("meta",{"name":"citation_fulltext_html_url"})

        if article is not None:
            article_dict['article']= article.text.strip()

        if title is not None:
            article_dict['title']= title['content'].strip()

        if vol is not None:
            article_dict['vol']= vol['content'].strip()

        if issue is not None:
            article_dict['issue']= issue['content'].strip()

        if author is not None:
            article_dict['author']= author['content'].strip()

        if institution is not None:
            article_dict['institution']= institution['content'].strip()

        if url is not None:
            article_dict['url']= url['content'].strip()

        json_data.append(article_dict)

df=pd.DataFrame(json_data)
df.to_csv('milbank2018.csv')

网站抓取文章-单独的合著者数据

1 个答案: