Question

我正在探索针对Firebase的Google Cloud Functions（启用Blaze支付定价计划）。

希望通过创建一个使用TMDB API并复制所有页面并将JSON数据存储到firebase数据库的项目进行测试

这是我的代码

// Array to hold async tasks
var asyncTasks = [];

var page = 0;
var total_pages = 630;

console.log('Started fetching Movies');
console.log('Total Pages ' + total_pages);

// Loop total no of pages times
for (var i = 0; i < total_pages; i++) {
    // We don't actually execute the async action here
    // We add a function containing it to an array of "tasks"
    asyncTasks.push(function (callback) {
        page++;
        console.log("Procesing Page " + page);

        var url = base_url + page;
        client.get(url, function (data, response) {

            // Get rateLimit from Raw Header
            var ratelimit = response.rawHeaders[27];

            // console.log(response.rawHeaders[26] + ' ' + response.rawHeaders[27]);
            if (ratelimit == 2) {
                // Rate Limit Exceeded wait 10 seconds
               console.log("Rate Limited Exceeded");

               // Sleep 10 Seconds to reset Rate Limit
               sleep.sleep(10);
            }

            global.all_data = global.all_data.concat(data.results);
            callback();
        }).on('error', function (err) {
            console.log('Something went wrong on the request', err.request.options);
            problem_pages.push(err.request.options.href);
            // Retrying once more before failure
            var eurl = err.request.options.href;
            client.get(eurl, function (data, response) {
                console.log("Retrying " + eurl);
                global.all_data = global.all_data.concat(data.results);
            });
        });
    });
}
const limit = functions.config().exec.limit;
console.log("Starting || execution [limit" + limit + "]");
async.parallelLimit(asyncTasks, limit, movie_results);
});

这是最后执行的movie_results函数

function movie_results () {
var moviesRef = defaultDatabase.ref("v1/movies");
var statusRef = defaultDatabase.ref("v1/status");

//moviesRef.remove();
var data_json = {};

console.log("Total Movies fetched " + global.all_data.length);

async.each(global.all_data,
    function(mov, cb) {
        data_json[mov.id] = mov;
        cb();
        },
    function() {
        console.log("Result post-processing done");
        console.log("Total =" + data_json.length + " | Expected =" + data_info.total_resu`enter code here`lts);
        console.log("Uploading to Firebase Datastore");
        moviesRef.update(data_json);

        console.log("Finished fetching movies");

        // Update status for this job on status DB
        var datim = new Date();
        statusRef.update({ datim: {"records_fetched": data_json.length, "records_expected": data_info.total_results,
                                   "total_pages": data_info.total_pages, "timestamp": datim}
                        });

        // Closing all db references
        moviesRef.off();
        statusRef.off();
    }
);
}

问题在于，大部分时间都可以执行...

正如您所看到的，我已经添加了日志记录，以便在执行asynTask中的函数时告诉我

console.log("Procesing Page " + page);

在Google Cloud日志中，我看到了

gcloud beta functions logs read --limit 20 --execution-id 118384524692947

我被卡在随机页面上（每当它卡住。）我理解这些功能仍然在BETA中。那么有人可以评论他们看到的问题和我一样。有没有办法检查当前是否有任何云功能进程执行，并知道是否已被杀死。我不认为它超时。感谢

I      tmdbUpdatePubSub  118384524692947  2017-04-26 09:47:22.286  Procesing Page 223
I      tmdbUpdatePubSub  118384524692947  2017-04-26 09:47:22.786  Procesing Page 224
I      tmdbUpdatePubSub  118384524692947  2017-04-26 09:47:23.384  Procesing Page 225
I      tmdbUpdatePubSub  118384524692947  2017-04-26 09:47:23.984  Procesing Page 226
I      tmdbUpdatePubSub  118384524692947  2017-04-26 09:47:25.084  Procesing Page 227
I      tmdbUpdatePubSub  118384524692947  2017-04-26 09:47:25.484  Procesing Page 228
I      tmdbUpdatePubSub  118384524692947  2017-04-26 09:47:26.184  Procesing Page 229
I      tmdbUpdatePubSub  118384524692947  2017-04-26 09:47:27.284  Procesing Page 230

Google Cloud功能无法响应（根据日志）

0 个答案: