Question

我正在使用一个使用Drive API将纯文本文件上传到Google云端硬盘的流程。即使实际的请求数量远远低于API控制台中设置的Drive API的每用户限制，该过程也经常会遇到速率限制异常。实际上，设置每用户限制似乎不会影响我们收到异常的速率。是否有一些其他限制（除了每用户限制）控制每秒可以发出多少请求？可以调整吗？

该过程对这些异常使用指数退避，因此操作最终会成功。我们每秒仅发出大约5个请求，每个用户限制设置为100。

Caused by: com.google.api.client.googleapis.json.GoogleJsonResponseException: 403 Forbidden
{
  "code" : 403,
  "errors" : [ {
    "domain" : "usageLimits",
    "message" : "Rate Limit Exceeded",
    "reason" : "rateLimitExceeded"
  } ],
  "message" : "Rate Limit Exceeded"
}

编辑：这是开发人员代码的“简化”版本。我们正在使用具有域委派的服务帐户，如：https://developers.google.com/drive/delegation所述。

package com.seto.fs.daemon;

import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.atomic.AtomicInteger;

import com.google.api.client.auth.oauth2.Credential;
import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
import com.google.api.client.googleapis.json.GoogleJsonResponseException;
import com.google.api.client.http.FileContent;
import com.google.api.client.http.HttpBackOffIOExceptionHandler;
import com.google.api.client.http.HttpBackOffUnsuccessfulResponseHandler;
import com.google.api.client.http.HttpBackOffUnsuccessfulResponseHandler.BackOffRequired;
import com.google.api.client.http.HttpRequest;
import com.google.api.client.http.HttpRequestInitializer;
import com.google.api.client.http.HttpResponse;
import com.google.api.client.http.HttpTransport;
import com.google.api.client.json.JsonFactory;
import com.google.api.client.json.jackson2.JacksonFactory;
import com.google.api.client.testing.util.MockBackOff;
import com.google.api.client.util.DateTime;
import com.google.api.client.util.ExponentialBackOff;
import com.google.api.services.drive.Drive;
import com.google.api.services.drive.Drive.Files.Insert;
import com.google.api.services.drive.DriveScopes;
import com.google.api.services.drive.model.ChildList;
import com.google.api.services.drive.model.ChildReference;
import com.google.api.services.drive.model.File.Labels;
import com.google.api.services.drive.model.ParentReference;


public class Test {
    private static final int testFilesCount = 100;
    private static final int threadsCount = 3;
    private static final AtomicInteger rateLimitErrorsCount = new AtomicInteger(0);

    private static final String impersonatedUser = "<impersonatedUserEmail>";
    private static final String serviceAccountID = "<some-id>@developer.gserviceaccount.com";
    private static final String serviceAccountPK = "/path/to/<public_key_fingerprint>-privatekey.p12";

    public static void main(String[] args) throws Exception {
        // Create HTTP transport
        HttpTransport httpTransport = GoogleNetHttpTransport.newTrustedTransport();
        // Create JsonFactory
        final JsonFactory jsonFactory = new JacksonFactory();

        // Create Google credential for service account
        final Credential credential = new GoogleCredential.Builder()
                .setTransport(httpTransport)
                .setJsonFactory(jsonFactory)
                .setServiceAccountScopes(Arrays.asList(DriveScopes.DRIVE))
                .setServiceAccountUser(impersonatedUser)
                .setServiceAccountId(serviceAccountID)
                .setServiceAccountPrivateKeyFromP12File(new File(serviceAccountPK))
                .build();

        // Create Drive client
        final Drive drive = new Drive.Builder(httpTransport, jsonFactory, new HttpRequestInitializer() {
            public void initialize(HttpRequest request) throws IOException {
                request.setContentLoggingLimit(0);
                request.setCurlLoggingEnabled(false);

                // Authorization initialization
                credential.initialize(request);

                // Exponential Back-off for 5xx response and 403 rate limit exceeded error
                HttpBackOffUnsuccessfulResponseHandler serverErrorHandler
                    = new HttpBackOffUnsuccessfulResponseHandler(new ExponentialBackOff.Builder().build());
                serverErrorHandler.setBackOffRequired(new BackOffRequired() {
                    public boolean isRequired(HttpResponse response) {
                        return response.getStatusCode() / 100 == 5
                            || (response.getStatusCode() == 403 && isRateLimitExceeded(
                                    GoogleJsonResponseException.from(jsonFactory, response)));
                    }
                });
                request.setUnsuccessfulResponseHandler(serverErrorHandler);

                // Back-off for socket connection error
                MockBackOff backOff = new MockBackOff();
                backOff.setBackOffMillis(2000);
                backOff.setMaxTries(5);
                request.setIOExceptionHandler(new HttpBackOffIOExceptionHandler(backOff));
            }
        }).setApplicationName("GoogleDriveUploadFile/1.0").build();

        // Get root folder id
        final String rootFolderId = drive.about().get().execute().getRootFolderId();

        // Query all children under root folder
        ChildList result = drive.children().list(rootFolderId).execute();

        // Delete all children under root folder
        for (ChildReference child : result.getItems()) {
            System.out.println("Delete child: " + child.getId());
            drive.files().delete(child.getId()).execute();
        }

        // Create a drive folder
        com.google.api.services.drive.model.File folderMetadata
            = new com.google.api.services.drive.model.File();
        folderMetadata.setMimeType("application/vnd.google-apps.folder")
            .setParents(Arrays.asList(new ParentReference().setId(rootFolderId)))
            .setTitle("DriveTestFolder");
        final com.google.api.services.drive.model.File driveTestFolder = drive.files().insert(folderMetadata).execute();

        // Create test files
        final List<File> testFiles = Collections.synchronizedList(createTestFiles());

        // Run threads to upload files to drive
        List<Thread> threads = new ArrayList<Thread>();
        for (int i = 0; i < threadsCount; i++) {
            Thread thread = new Thread(new Runnable() {
                public void run() {
                    while (testFiles.size() > 0) {
                        try {
                            File testFile = testFiles.remove(0);

                            // The file meta data
                            com.google.api.services.drive.model.File fileMetadata =
                                new com.google.api.services.drive.model.File()
                                .setTitle(testFile.getName()).setParents(Arrays.asList(new ParentReference().setId(driveTestFolder.getId())))
                                .setLabels(new Labels().setRestricted(false)).setMimeType("text/plain")
                                .setModifiedDate(new DateTime(testFile.lastModified()))
                                .setDescription("folder:MyDrive " + testFile.getName());

                            // Insert to drive
                            FileContent fileContent = new FileContent("text/plain", testFile);
                            Insert insertFileCommand = drive.files().insert(fileMetadata, fileContent)
                                    .setUseContentAsIndexableText(true);
                            insertFileCommand.getMediaHttpUploader().setDirectUploadEnabled(true);

                            insertFileCommand.execute();

                            System.out.println(testFile.getName() + " is uploaded");
                        } catch (IOException e) {
                            e.printStackTrace();
                        } catch (IndexOutOfBoundsException e) {
                            // ignore
                        }
                    }
                }
            });
            threads.add(thread);
        }

        long startTime = System.currentTimeMillis();

        for (Thread thread : threads) {
            thread.start();
        }

        for (Thread thread : threads) {
            thread.join();
        }

        System.out.println("Total time spent: " + (System.currentTimeMillis() - startTime)
                + "ms for " + testFilesCount + " files with " + threadsCount + " threads");
        System.out.println("Rate limit errors hit: " + rateLimitErrorsCount.intValue());
    }

    private static List<File> createTestFiles() throws Exception {

        // Create test files directory
        File testFolder = new File("TestFiles");
        testFolder.mkdir();

        // Create test files
        List<File> testFiles = new ArrayList<File>();
        for (int i = 0; i < testFilesCount; i++) {
            File testFile = new File("TestFiles/" + i + ".txt");
            FileOutputStream fops = new FileOutputStream(testFile);
            fops.write(testFile.getAbsolutePath().getBytes());
            fops.close();
            testFiles.add(testFile);
        }
        return testFiles;
    }

    private static boolean isRateLimitExceeded(GoogleJsonResponseException ex) {
        boolean result = false;
        if (ex.getDetails() != null && ex.getDetails().getErrors() != null
                && ex.getDetails().getErrors().size() > 0) {
            String reason = ex.getDetails().getErrors().get(0).getReason();
            result = "rateLimitExceeded".equals(reason) || "userRateLimitExceeded".equals(reason);
            if (result) {
                rateLimitErrorsCount.incrementAndGet();
                System.err.println("Rate limit error");
            }
        }

        return result;
    }

}

编辑：当我们使用单个线程并在每次调用之间放置500毫秒的延迟时，我们遇到了这个异常。看起来不可能在我们配置的每用户速率附近。即使是每秒默认的10个请求也是不可能的。为什么呢？

Answer 1

上传有一个下限，它是所有应用中的每位用户。将看看我们是否可以在文档中发布限制。

Answer 2

这意味着您的配置出现问题。您可以使用低配额的Drive API，就像您遇到的API配置错误一样。

请检查您是否在APIs Console＆gt;中启用了Drive API服务
请检查您是否使用了正确的API密钥或授权流程。

如果您仍然无法找到导致此错误的原因，请附上最小代码以重现此错误。

Answer 3

跟进Steve Bazyl的回答......

此Google API异常是由C＃ETL程序生成的，该程序将单个表单行请求加载到SQL数据库表中。

ex = {＆＃34; Google.Apis.Requests.RequestError \ r \ n不足的令牌 quota＆＃39; ReadGroup＆＃39;并限制用户100＆＃39;服务＆＃39; sheets.googleapis.com＆＃39;对于消费者......

＆＃34; USER-100s ＆＃34;限制似乎与Google API v4 "Usage Limits" page中的以下文字有关。

此版本的Google表格API每次限制为500个每个项目100秒，每100秒100次请求用户即可。分别跟踪读取和写入的限制。有没有每日使用限制。

基本上，需要一个限制机制来避免每个时间单位约束的这些请求，减少请求或两者的组合。

此Google page还提到可以使用＆＃34;结算帐户＆＃34;来增加这些配额限制。

提高每个用户的Google Drive API限制不会阻止速率限制例外

3 个答案: