Question

我尝试使用排序算法创建一个网络抓取工具，如果页面排名显示基本想法，现在它可以正常工作，但是当我尝试抓取其他网页时，它并没有。我认为问题符合要求

for link in soup.findAll ('a', {'class':' '}):"

但如果我删除：

 , {'class':' '}

它给了我一堆错误。其他问题是如果我为“网址”设置了其他网页。它运行但没有回馈任何东西。

 import requests
from bs4 import BeautifulSoup
from collections import defaultdict
from operator import itemgetter

all_links = defaultdict(int)

def webpages():

        url = 'http://www.hm.com/gb/department/HOME'
        source_code = requests.get(url)
        text = source_code.text
        soup = BeautifulSoup(text)
        for link in soup.findAll ('a', {'class':' '}):
            href = link.get('href')
            print(href)
            get_single_item_data(href)
        return all_links

def get_single_item_data(item_url):
    source_code = requests.get(item_url)
    text = source_code.text
    soup = BeautifulSoup(text)
    for link in soup.findAll('a'):
        href = link.get('href')
        if href and href.startswith('http://www.'):
            if href:
                all_links[href] += 1
            print(href)


def sort_algorithm(list):
    for index in range(1,len(list)):
        value= list[index]
        i = index - 1
        while i>=0:
            if value < list[i]:
                list[i+1] = list[i]
                list[i] = value
                i=i -1
            else:
                break

units = ["", "one", "two", "three", "four", "five",
         "six", "seven", "eight", "nine "]
teens = ["", "eleven", "twelve", "thirteen", "fourteen",
         "fifteen", 'sixteen', "seventeen", "eighteen", "nineteen"]
tens = ["", "ten", "twenty", "thirty", "forty",
        "fifty", "sixty", "seventy", "eighty", "ninety"]
thousands = ["", "thousand", "million", "billion", "trillion",
             "quadrillion", "quintillion", "sextillion", "septillion", "octillion",
             "nonillion", "decillion", "undecillion", "duodecillion", "tredecillion",
             "quattuordecillion", "sexdecillion", "septendecillion", "octodecillion",
             "novemdecillion", "vigintillion "]


def num_to_words(n):
    words = []
    if n == 0:
        words.append("zero")
    else:
        num_str = "{}".format(n)
        groups = (len(num_str) + 2) // 3
        num_str = num_str.zfill(groups * 3)
        for i in range(0, groups * 3, 3):
            h = int(num_str[i])
            t = int(num_str[i + 1])
            u = int(num_str[i + 2])
            print()
            print(units[i])
            g = groups - (i // 3 + 1)
            if h >= 1:
                words.append(units[h])
                words.append("hundred")
                if int(num_str) % 100:
                    words.append("and")
            if t > 1:
                words.append(tens[t])
                if u >= 1:
                    words.append(units[u])
            elif t == 1:
                if u >= 1:
                    words.append(teens[u])
                else:
                    words.append(tens[t])
            else:
                if u >= 1:
                    words.append(units[u])

            if g >= 1 and (h + t + u) > 0:
                words.append(thousands[g])
    return " ".join(words)

webpages()

for k, v in sorted(webpages().items(),key=itemgetter(1),reverse=True):
    print(k, num_to_words(v))

在python中创建web爬虫

0 个答案: