我尝试在用户运行查询时静态向用户提供csv文件。我有一个工作人员在后台运行它在Redis上。看起来数据存在,但链接转到404页面。我是否必须在AWS / S3或类似服务上托管这些文件以使该链接起作用?
我使用的是Python,Tornado,Redis和Heroku。
主要Python运行。
class MainHandler(tornado.web.RequestHandler):
debug=True
static_hash_cache=False
serve_traceback=True
autoreload=True
def get(self):
# product = self.get_argument('product-type', '')
apikey = self.get_argument('apikey', '')
email = self.get_argument('email', '')
domain = self.get_argument('domain', '')
start = self.get_argument('start', '')
end = self.get_argument('end', '')
urls = domain.split(",")
print "urls: ", urls
url_to_filenames_dictionary = {}
for u in urls:
print 'url', u
filePath = os.path.join(".", "static", "{0}{1}{2}.csv".format(domain, start, end))
if apikey:
#results = max_concurrents(apikey, u, start, end, save_to=True)
enqueue_job(apikey, u, start, end, save_to=True)
if filePath:
#url_to_filenames_dictionary[u] = results
url_to_filenames_dictionary[u] = filePath
print filePath + 'valid'
self.render('index.html', data=str(url_to_filenames_dictionary), start=start, end=end, urls=domain.split(","), apikey=apikey)
def post(self):
apikey = self.get_argument('apikey','')
email = self.get_argument('email', '')
domain = self.get_argument('domain','')
start = self.get_argument('start','')
end = self.get_argument('end','')
urls = domain.split(",")
for u in urls:
filePath = u + "_" + start + "_" + end + ".csv"
print filePath
print r
print email
def make_app():
debug=True
static_hash_cache=False
serve_traceback=True
autoreload=True
return tornado.web.Application([
(r"/", MainHandler),
(r"/static", tornado.web.StaticFileHandler,
dict(path=settings['static_path'])),
], **settings)
settings = {
"static_path": os.path.join(os.path.dirname(__file__), "static"),
}
def enqueue_job(apikey, domain, start, end, save_to=True):
try:
q = Queue(connection=conn)
result = q.enqueue_call(func=max_concurrents, args=(apikey, domain, start, end, save_to))
print result
except Exception as e:
raise e
if __name__ == "__main__":
app = make_app()
port = int(os.environ.get('PORT', 5000))
app.listen(port, '0.0.0.0')
print "It's alive!!!"
tornado.ioloop.IOLoop.current().start()
下面的工作是由工人来完成的:
def max_concurrents(apikey, domain, start, end, save_to=False):
d_start = datetime.strptime(start, "%Y-%m-%d")
d_end = datetime.strptime(end, "%Y-%m-%d")
delta = d_end - d_start
toReturn = []
rows = []
# grab the domain from comma separated list and make separate async requests for each domain
urls = domain.split(",")
print urls
for u in urls:
print u
for i in xrange(delta.days + 1):
curr_date = d_start + timedelta(days=i)
formatted_date = curr_date.strftime("%Y-%m-%d")
params = {
'host': u,
'apikey': apikey,
'date': formatted_date,
}
r = requests.get(REPORTS_API_ENDPOINT, params=params)
data = r.json()
row = [u, formatted_date, str(data['data']['overview']['data']['max_concurrents']['num'])]
toReturn.append(','.join(row))
rows.append(row)
#print r.json()
if save_to:
path = os.path.join(".", "static", "{0}{1}{2}.csv".format(domain, start, end))
print start, end
print "THIS IS THE FILEPATH: ", path
print "end"
with open(path, 'wb') as csvFile:
writer = DictWriter(csvFile, fieldnames=['u', 'date', 'max_concurrents'])
writer.writeheader()
for u, date, max_concurrents in rows:
writer.writerow({
'u': u,
'date': date,
'max_concurrents': max_concurrents,
})
return '\n'.join(toReturn)
工人:
import os
import redis
from rq import Worker, Queue, Connection
listen = ['high', 'default', 'low']
redis_url = os.getenv('REDIS_URL', 'redis://localhost:6379')
conn = redis.from_url(redis_url)
if __name__ == '__main__':
print "worker is working!!!"
with Connection(conn):
worker = Worker(map(Queue, listen))
worker.work()