尝试运行Teonite_project / web_scrapper.py脚本时遇到此错误:
File "C:/Users/kfhei/Desktop/Teonite_project/Teonite_project/web_scrapper.py", line 9, in <module>
django.setup()
File "C:\Users\kfhei\Desktop\Teonite_project\env\lib\site-packages\django\__init__.py", line 19, in setup
configure_logging(settings.LOGGING_CONFIG, settings.LOGGING)
File "C:\Users\kfhei\Desktop\Teonite_project\env\lib\site-packages\django\conf\__init__.py", line 56, in __getattr__
self._setup(name)
File "C:\Users\kfhei\Desktop\Teonite_project\env\lib\site-packages\django\conf\__init__.py", line 43, in _setup
self._wrapped = Settings(settings_module)
File "C:\Users\kfhei\Desktop\Teonite_project\env\lib\site-packages\django\conf\__init__.py", line 106, in __init__
mod = importlib.import_module(self.SETTINGS_MODULE)
File "C:\Users\kfhei\Desktop\Teonite_project\env\lib\importlib\__init__.py", line 126, in import_module
return _bootstrap._gcd_import(name[level:], package, level)
File "<frozen importlib._bootstrap>", line 994, in _gcd_import
File "<frozen importlib._bootstrap>", line 971, in _find_and_load
File "<frozen importlib._bootstrap>", line 955, in _find_and_load_unlocked
File "<frozen importlib._bootstrap>", line 665, in _load_unlocked
File "<frozen importlib._bootstrap_external>", line 678, in exec_module
File "<frozen importlib._bootstrap>", line 219, in _call_with_frames_removed
File "C:\Users\kfhei\Desktop\Teonite_project\Teonite_project\Teonite_project\settings.py", line 15, in <module>
django.setup()
File "C:\Users\kfhei\Desktop\Teonite_project\env\lib\site-packages\django\__init__.py", line 19, in setup
configure_logging(settings.LOGGING_CONFIG, settings.LOGGING)
File "C:\Users\kfhei\Desktop\Teonite_project\env\lib\site-packages\django\conf\__init__.py", line 56, in __getattr__
self._setup(name)
File "C:\Users\kfhei\Desktop\Teonite_project\env\lib\site-packages\django\conf\__init__.py", line 43, in _setup
self._wrapped = Settings(settings_module)
File "C:\Users\kfhei\Desktop\Teonite_project\env\lib\site-packages\django\conf\__init__.py", line 125, in __init__
raise ImproperlyConfigured("The SECRET_KEY setting must not be empty.")
django.core.exceptions.ImproperlyConfigured: The SECRET_KEY setting must not be empty.
我的脚本:
from bs4 import BeautifulSoup
from urllib.request import Request, urlopen
#import simplejson as json
import os
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "Teonite_project.settings")
import django
django.setup()
from words.models import Word
from authors.models import Author
URL_DOMAIN = 'https://teonite.com/blog/'
def get_links(url):
'''
Returning the array of links
to blog articles from the website
'''
html = get_html(url)
links = []
for link in html.findAll('a'):
link = link.get('href')
if link == None or link[6:9] == 'tag' or link[6:10]=='page':
pass
elif link[:6] == '/blog/':
link = url[:19] + link
links.append(link)
return links
def get_html(url):
'''
Returning the HTML of the website
'''
req = Request(url)
html_page = urlopen(req)
html = BeautifulSoup(html_page, "html.parser")
return html
def get_text(url):
'''
Extracting the post content of
the articles from the blog
'''
html = get_html(url)
text =''
for content in html.select('.post-content'):
text = text + content.text
return content.text
def get_author(url):
'''
Extracting the name of the Author
from the articles
'''
html = get_html(url)
for author in html.select('.author-content'):
return author.text
if __name__ == '__main__':
'''
Main function tasks:
* Extract the neccessary data from the website,
* Save it to the database
'''
links = get_links(URL_DOMAIN)
author_dict_database = {}
word_dict_database = {}
for link in links:
text = get_text(link).strip()
wordslist = text.split()
author = get_author(link).strip()
for word in wordslist:
if not word.isalpha():
wordslist.remove(word)
word = word.lower()
if author in author_dict_database:
for word in wordslist:
if word in word_dict_database:
author_dict_database[author][word] += 1
else:
author_dict_database[author][word] = 1
else:
for word in wordslist:
if word in word_dict_database:
word_dict_database[word] += 1
else:
word_dict_database[word] = 1
author_dict_database[author] = word_dict_database
#Saving values to postgres database
for key_author,word_dict in author_dict_database:
database_author = Author(author=key_author)
for key_word, value_count in word_dict:
database_word = Word(author=key_author, words_list=key_word, words_count=value_count)
如您所见,我尝试了不同的方法来使其工作。我已经阅读了stackoverflow中的几个主题,并尝试在不同的网站中进行搜索。 我还配置了wsgi.py文件
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "Teonite_project.settings")
不幸的是,我不知道为什么会这样,因为我已经在我的Teonite_project / Teonite_project / settings.py中设置了SECRET_KEY
基本上我只想运行我的脚本,并将报废的值添加到我的作者和单词模型中的postgres数据库中。