我正在探索针对Firebase的Google Cloud Functions(启用Blaze支付定价计划)。
希望通过创建一个使用TMDB API并复制所有页面并将JSON数据存储到firebase数据库的项目进行测试
这是我的代码
// Array to hold async tasks
var asyncTasks = [];
var page = 0;
var total_pages = 630;
console.log('Started fetching Movies');
console.log('Total Pages ' + total_pages);
// Loop total no of pages times
for (var i = 0; i < total_pages; i++) {
// We don't actually execute the async action here
// We add a function containing it to an array of "tasks"
asyncTasks.push(function (callback) {
page++;
console.log("Procesing Page " + page);
var url = base_url + page;
client.get(url, function (data, response) {
// Get rateLimit from Raw Header
var ratelimit = response.rawHeaders[27];
// console.log(response.rawHeaders[26] + ' ' + response.rawHeaders[27]);
if (ratelimit == 2) {
// Rate Limit Exceeded wait 10 seconds
console.log("Rate Limited Exceeded");
// Sleep 10 Seconds to reset Rate Limit
sleep.sleep(10);
}
global.all_data = global.all_data.concat(data.results);
callback();
}).on('error', function (err) {
console.log('Something went wrong on the request', err.request.options);
problem_pages.push(err.request.options.href);
// Retrying once more before failure
var eurl = err.request.options.href;
client.get(eurl, function (data, response) {
console.log("Retrying " + eurl);
global.all_data = global.all_data.concat(data.results);
});
});
});
}
const limit = functions.config().exec.limit;
console.log("Starting || execution [limit" + limit + "]");
async.parallelLimit(asyncTasks, limit, movie_results);
});
这是最后执行的movie_results函数
function movie_results () {
var moviesRef = defaultDatabase.ref("v1/movies");
var statusRef = defaultDatabase.ref("v1/status");
//moviesRef.remove();
var data_json = {};
console.log("Total Movies fetched " + global.all_data.length);
async.each(global.all_data,
function(mov, cb) {
data_json[mov.id] = mov;
cb();
},
function() {
console.log("Result post-processing done");
console.log("Total =" + data_json.length + " | Expected =" + data_info.total_resu`enter code here`lts);
console.log("Uploading to Firebase Datastore");
moviesRef.update(data_json);
console.log("Finished fetching movies");
// Update status for this job on status DB
var datim = new Date();
statusRef.update({ datim: {"records_fetched": data_json.length, "records_expected": data_info.total_results,
"total_pages": data_info.total_pages, "timestamp": datim}
});
// Closing all db references
moviesRef.off();
statusRef.off();
}
);
}
问题在于,大部分时间都可以执行...
正如您所看到的,我已经添加了日志记录,以便在执行asynTask中的函数时告诉我
console.log("Procesing Page " + page);
在Google Cloud日志中,我看到了
gcloud beta functions logs read --limit 20 --execution-id 118384524692947
我被卡在随机页面上(每当它卡住。)我理解这些功能仍然在BETA中。那么有人可以评论他们看到的问题和我一样。有没有办法检查当前是否有任何云功能进程执行,并知道是否已被杀死。我不认为它超时。感谢
I tmdbUpdatePubSub 118384524692947 2017-04-26 09:47:22.286 Procesing Page 223
I tmdbUpdatePubSub 118384524692947 2017-04-26 09:47:22.786 Procesing Page 224
I tmdbUpdatePubSub 118384524692947 2017-04-26 09:47:23.384 Procesing Page 225
I tmdbUpdatePubSub 118384524692947 2017-04-26 09:47:23.984 Procesing Page 226
I tmdbUpdatePubSub 118384524692947 2017-04-26 09:47:25.084 Procesing Page 227
I tmdbUpdatePubSub 118384524692947 2017-04-26 09:47:25.484 Procesing Page 228
I tmdbUpdatePubSub 118384524692947 2017-04-26 09:47:26.184 Procesing Page 229
I tmdbUpdatePubSub 118384524692947 2017-04-26 09:47:27.284 Procesing Page 230