Question

我想下载大型科学抽象数据，让我们说一下2000 Pubmed ID。我的python代码很草率，看起来很慢。有没有快速有效的方法来收集这些摘要？

如果这是最快的方法，我如何衡量它，以便我能够与其他人或家庭对比工作情况（不同的ISP可能参与速度）？

附上我的代码。

import sqlite3
from Bio.Entrez import read,efetch,email,tool
from metapub import PubMedFetcher
import pandas as pd
import requests
from datetime import  date
import xml.etree.ElementTree as ET
import time
import sys
reload(sys)
sys.setdefaultencoding('utf8')
Abstract_data = pd.DataFrame(columns=["name","pmid","abstract"])

def abstract_download(self,dict_pmids):
    """
        This method returns abstract for a given pmid and add to the abstract data
    """
    index=0
    baseUrl = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/"
    for names in dict_pmids:
        for pmid in dict_pmids[names]:
            try:
                abstract = []
                url = baseUrl+"efetch.fcgi?db=pubmed&id="+pmid+"&rettype=xml"+
                response=requests.request("GET",url,timeout=500).text
                response=response.encode('utf-8')
                root=ET.fromstring(response)
                root_find=root.findall('./PubmedArticle/MedlineCitation/Article/Abstract/')
                if len(root_find)==0:
                    root_find=root.findall('./PubmedArticle/MedlineCitation/Article/ArticleTitle')
                for i in range(len(root_find)):
                    if root_find[i].text != None:
                        abstract.append(root_find[i].text)
                if abstract is not None:
                    Abstract_data.loc[index]=names,pmid,"".join(abstract)
                index+=1
            except:
                print "Connection Refused"
                time.sleep(5)
                continue
    return Abstract_data

编辑：此脚本发生的一般错误似乎是“连接被拒绝”。请参阅下面的ZF007答案，了解这是如何解决的。

Answer 1

以下代码有效。您的脚本依赖于格式错误的URL构造。此外，如果脚本内部出现问题，则响应是拒绝连接。实际情况并非如此，因为正是代码处理了检索到的数据。我已经做了一些调整以使代码为我工作并留下评论，因为缺乏需要调整的地方dict_pmids列表。

#!/usr/bin/env python
# -*- coding: utf-8 -*-

import sys, time, requests, sqlite3

import pandas as pd
import xml.etree.ElementTree as ET

from metapub import PubMedFetcher
from datetime import  date
from Bio.Entrez import read,efetch,email,tool


def abstract_download(pmids):

    """
        This method returns abstract for a given pmid and add to the abstract data
    """

    index   = 0
    baseUrl = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/"
    collected_abstract = []

    # code below diabled to get general abstract extraction from pubmed working. I don't have the dict_pmid list.

    """
    for names in dict_pmids:
        for pmid in dict_pmids[names]:

    move below working code to the right to get it in place with above two requirements prior to providing dict_pmid list.

    # from here code works upto the next comment. I don't have the dict_pmid list.
    """

    for pmid in pmids:

        print 'pmid : %s\n' % pmid

        abstract  = []
        root = ''

        try:

            url       = '%sefetch.fcgi?db=pubmed&id=%s&rettype=xml' % (baseUrl, pmid)
             # checks my url... line to parse into a webbrowser like firefox.
            print 'url', url

            response  = requests.request("GET", url, timeout=500).text
             # check if I got a response.
            print 'response', response

#                response  = response.encode('utf-8')
            root      = ET.fromstring(response)

        except Exception as inst:
            # besides a refused connection.... the "why" it was connected comes in handly to resolve issues at hand
            # if and when they happen.
            print "Connection Refused", inst

            time.sleep(5)
            continue

        root_find = root.findall('./PubmedArticle/MedlineCitation/Article/Abstract/')

        if len(root_find)==0:

                root_find = root.findall('./PubmedArticle/MedlineCitation/Article/ArticleTitle')

        # check if I found something
        print 'root_find : %s\n\n' % root_find

        for i in range(len(root_find)):

            if root_find[i].text != None:

                abstract.append(root_find[i].text)

        Abstract_data = pd.DataFrame(columns=["name","pmid","abstract"])

        # check if I found something
        #print 'abstract : %s\n' % abstract

        # code works up to the print statement ''abstract', abstract' teh rest is disabled because I don't have the dict_pmid list.

        if abstract is not None:

#                Abstract_data.loc[index] = names,pmid,"".join(abstract)

            index += 1
            collected_abstract.append(abstract)

    # change back return Abstract_data when dict_pmid list is administered.
#    return Abstract_data
    return collected_abstract

if __name__ == '__main__':

    sys.stdout.flush()

    reload(sys)
    sys.setdefaultencoding('utf8')

    pubmedIDs = range(21491000, 21491001)

    mydata = abstract_download(pubmedIDs)

    print 'mydata : %s' % (mydata)

是否有任何快速有效的方式来获取被发表的摘要？

1 个答案: