我是否必须在AWS上托管用于heroku应用程序的csv文件?

时间:2016-06-09 18:50:11

标签: python heroku redis tornado tornado-redis

我尝试在用户运行查询时静态向用户提供csv文件。我有一个工作人员在后台运行它在Redis上。看起来数据存在,但链接转到404页面。我是否必须在AWS / S3或类似服务上托管这些文件以使该链接起作用?

我使用的是Python,Tornado,Redis和Heroku。

主要Python运行。

    class MainHandler(tornado.web.RequestHandler):
    debug=True
    static_hash_cache=False
    serve_traceback=True
    autoreload=True
    def get(self):
        # product = self.get_argument('product-type', '')
        apikey = self.get_argument('apikey', '')
        email = self.get_argument('email', '')
        domain = self.get_argument('domain', '')
        start = self.get_argument('start', '')
        end = self.get_argument('end', '')
        urls = domain.split(",")
        print "urls: ", urls

        url_to_filenames_dictionary = {}
        for u in urls:
            print 'url', u
            filePath = os.path.join(".", "static", "{0}{1}{2}.csv".format(domain, start, end))
            if apikey:
                #results = max_concurrents(apikey, u, start, end, save_to=True)
                enqueue_job(apikey, u, start, end, save_to=True)
                if filePath:
                    #url_to_filenames_dictionary[u] = results
                    url_to_filenames_dictionary[u] = filePath
                    print filePath + 'valid'

        self.render('index.html', data=str(url_to_filenames_dictionary), start=start, end=end, urls=domain.split(","), apikey=apikey)

    def post(self):
        apikey = self.get_argument('apikey','')
        email = self.get_argument('email', '')
        domain = self.get_argument('domain','')
        start = self.get_argument('start','')
        end = self.get_argument('end','')
        urls = domain.split(",")
        for u in urls:
            filePath = u + "_"  + start + "_"  + end + ".csv"
            print filePath
            print r
            print email

def make_app():
    debug=True
    static_hash_cache=False
    serve_traceback=True
    autoreload=True
    return tornado.web.Application([
        (r"/", MainHandler),
        (r"/static", tornado.web.StaticFileHandler,
        dict(path=settings['static_path'])),
    ], **settings)

settings = {
    "static_path": os.path.join(os.path.dirname(__file__), "static"),
}


def enqueue_job(apikey, domain, start, end, save_to=True):
    try:
        q = Queue(connection=conn)
        result = q.enqueue_call(func=max_concurrents, args=(apikey, domain, start, end, save_to))
        print result
    except Exception as e:
        raise e


if __name__ == "__main__":
    app = make_app()
    port = int(os.environ.get('PORT', 5000))
    app.listen(port, '0.0.0.0')
    print "It's alive!!!"
    tornado.ioloop.IOLoop.current().start()

下面的工作是由工人来完成的:

def max_concurrents(apikey, domain, start, end, save_to=False):

    d_start = datetime.strptime(start, "%Y-%m-%d")
    d_end = datetime.strptime(end, "%Y-%m-%d")

    delta = d_end - d_start

    toReturn = []
    rows = []
    # grab the domain from comma separated list and make separate async requests for each domain

    urls = domain.split(",")

    print urls

    for u in urls:
        print u
        for i in xrange(delta.days + 1):
            curr_date = d_start + timedelta(days=i)
            formatted_date = curr_date.strftime("%Y-%m-%d")
            params = {
                'host': u,
                'apikey': apikey,
                'date': formatted_date,
            }

            r = requests.get(REPORTS_API_ENDPOINT, params=params)

            data = r.json()

            row = [u, formatted_date, str(data['data']['overview']['data']['max_concurrents']['num'])]

            toReturn.append(','.join(row))
            rows.append(row)
            #print r.json()
        if save_to:
            path = os.path.join(".", "static", "{0}{1}{2}.csv".format(domain, start, end))
            print start, end
            print "THIS IS THE FILEPATH: ", path
            print "end"

            with open(path, 'wb') as csvFile:
                writer = DictWriter(csvFile, fieldnames=['u', 'date', 'max_concurrents'])
                writer.writeheader()
                for u, date, max_concurrents in rows:
                    writer.writerow({
                        'u': u,
                        'date': date,
                        'max_concurrents': max_concurrents,
                    })

        return '\n'.join(toReturn)

工人:

import os

import redis
from rq import Worker, Queue, Connection


listen = ['high', 'default', 'low']

redis_url = os.getenv('REDIS_URL', 'redis://localhost:6379')

conn = redis.from_url(redis_url)


if __name__ == '__main__':
    print "worker is working!!!"
    with Connection(conn):
        worker = Worker(map(Queue, listen))
        worker.work()

0 个答案:

没有答案