线程在Storm Cluster中不起作用

时间:2013-04-26 17:05:02

标签: python multithreading queue apache-storm

我正在尝试执行以下操作,在将创建队列的单独线程中启动流侦听器,稍后将处理这些队列...但是Storm在线程之后不执行任何操作。它停留在那边。

我的代码看起来像这样:

import os, sys, traceback, random, StringIO, time
import random
from uuid import uuid4
from select import select
from subprocess import Popen,PIPE
import pyinotify
import simplejson, pycurl
import sys, signal
import twitter
import tweepy
import Queue
import threading
try:
    import simplejson as json
except ImportError:
    import json

import storm
queue = Queue.Queue()

class MyModelParser(tweepy.parsers.ModelParser):
    def parse(self, method, payload):
        result = super(MyModelParser, self).parse(method, payload)
        result._payload = json.loads(payload)
        return result

class CustomStreamListener(tweepy.StreamListener):
    ''' Handles data received from the stream. '''
    def __init__(self, api, q):
        self.api = api
        self.queue = q
        self.queue.put('lalala')

    def on_status(self, status):
        self.queue.put('%s' % status.author.screen_name)
        self.queue.task_done()

    def on_error(self, status_code):
        return True # To continue listening

    def on_timeout(self):
        return True # To continue listening

class Starter():
    def __init__(self,q):
        self.queue = q
        hashtag = ['justinbieber','snooki','daddy_yankee','MikeTyson','iamdiddy','lala']
        auth = self.t_auth()
        api = tweepy.API(auth, parser=MyModelParser())
        stream = tweepy.streaming.Stream(auth,CustomStreamListener(api,queue))
        stream.filter(follow=None, track=hashtag)

    def t_auth(self):
        consumer_key=""
        consumer_secret=""
        access_key = ""
        access_secret = ""

        auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
        auth.set_access_token(access_key, access_secret) 

        return auth 

class TwitterSpout(storm.Spout):
    SPOUT_NAME = "TwitterSpout"
    queue = queue

    def initialize(self, conf, context):
        self.pid = os.getpid()      
        try:
            t = threading.Thread(target=Starter(self.queue) )
            t.daemon=True
            t.start()           

        except KeyboardInterrupt, e:
            self.log('\n\nStopping')
            raise 

0 个答案:

没有答案