我为scrapy写了一个自定义命令:
from pyes import *
from eScraperInterface import settings
from django.core.management import setup_environ
setup_environ(settings)
from eScraperInterfaceApp.models import scrapedData
from eScraperInterfaceApp.eScraper_utils import eScraperUtils
#------------------------------------------------------------------------------
class indexData():
def __init__(self):
self.utilsOBJ = eScraperUtils()
conn = ES('127.0.0.1:9200')
conn.create_index("scraped_data")
def run(self):
for i in scrapedData.objects.filter():
data = {'productTitle':i.productTitle,'productSite':i.productSite,
'productURL':i.productURL,'productDesc':i.productDesc,
'productCategory':i.productCategory,'productSubCategory':i.productSubCategory,
'productMRP':i.productMRP,'productPrice':i.productPrice,'hasVariants':i.hasVariants,
'availability':i.availability,'currency':i.currency,'image_paths':i.image_paths}
a = self.utilsOBJ.create_Index(data)
print a
当我尝试使用scrapy shell命令时出现此错误:
Traceback (most recent call last):
File "/usr/local/bin/scrapy", line 4, in <module>
execute()
File "/usr/local/lib/python2.7/dist-packages/scrapy/cmdline.py", line 111, in execute
cmds = _get_commands_dict(settings, inproject)
File "/usr/local/lib/python2.7/dist-packages/scrapy/cmdline.py", line 37, in _get_commands_dict
cmds.update(_get_commands_from_module(cmds_module, inproject))
File "/usr/local/lib/python2.7/dist-packages/scrapy/cmdline.py", line 27, in _get_commands_from_module
for cmd in _iter_command_classes(module):
File "/usr/local/lib/python2.7/dist-packages/scrapy/cmdline.py", line 18, in _iter_command_classes
for module in walk_modules(module_name):
File "/usr/local/lib/python2.7/dist-packages/scrapy/utils/misc.py", line 65, in walk_modules
submod = __import__(fullpath, {}, {}, [''])
File "/home/vaibhav/scrapyprog/comparison/eScraperInterface/eScraper/commands/indexData.py", line 10, in <module>
from eScraperInterfaceApp.models import scrapedData
File "/home/vaibhav/scrapyprog/comparison/eScraperInterface/eScraperInterfaceApp/models.py", line 3, in <module>
from django.db import models
File "/usr/local/lib/python2.7/dist-packages/django/db/__init__.py", line 40, in <module>
backend = load_backend(connection.settings_dict['ENGINE'])
File "/usr/local/lib/python2.7/dist-packages/django/db/__init__.py", line 34, in __getattr__
return getattr(connections[DEFAULT_DB_ALIAS], item)
File "/usr/local/lib/python2.7/dist-packages/django/db/utils.py", line 92, in __getitem__
backend = load_backend(db['ENGINE'])
File "/usr/local/lib/python2.7/dist-packages/django/db/utils.py", line 24, in load_backend
return import_module('.base', backend_name)
File "/usr/local/lib/python2.7/dist-packages/django/utils/importlib.py", line 35, in import_module
__import__(name)
File "/usr/local/lib/python2.7/dist-packages/django/db/backends/mysql/base.py", line 30, in <module>
from django.db import utils
ImportError: cannot import name utils
答案 0 :(得分:0)
你在系统上安装了python-mysqldb吗? sudo apt-get install python-mysqldb 或者pip install mysqldb