s3客户端get_object(Range = byte_range)未能检索对象的第一个字节

时间:2018-12-17 20:26:17

标签: amazon-s3 boto3 pivotal-cloud-foundry botocore

以下代码中修复的所有变量均仅在尝试返回早期字节范围(例如bytes = 0-1000)时返回“ SignatureDoesNotMatch”异常。如果范围较大(例如bytes = 400000-500000),则不会发生例外情况

using System.IO;

public void UploadFile2()
    {
        FilePath = Application.dataPath + "/StreamingAssets/ARITMETICA3DAZ1.txt";
        Debug.Log("Path: " + FilePath);

    Uri uri = new Uri(FTPHost + "/" + new FileInfo(FilePath).Name);
    FtpState state = new FtpState();
    FtpWebRequest request = (FtpWebRequest)WebRequest.Create(uri);
    request.Method = WebRequestMethods.Ftp.UploadFile;

    request.Credentials = new NetworkCredential (FTPUserName,FTPPassword);

    state.Request = request;
    state.FileName = FilePath;

    request.BeginGetRequestStream(
        //new AsyncCallback (EndGetStreamCallback), 
        new AsyncCallback (EndGetStreamCallback), 
        state
    );




    public class FtpState
    {
        private ManualResetEvent wait;
        private FtpWebRequest request;
        private string fileName;
        private Exception operationException = null;
        string status;

        public FtpState()
        {
            wait = new ManualResetEvent(false);
        }

        public ManualResetEvent OperationComplete
        {
            get {return wait;}
        }

        public FtpWebRequest Request
        {
            get {return request;}
            set {request = value;}
        }

        public string FileName
        {
            get {return fileName;}
            set {fileName = value;}
        }
        public Exception OperationException
        {
            get {return operationException;}
            set {operationException = value;}
        }
        public string StatusDescription
        {
            get {return status;}
            set {status = value;}
        }
    }
    public class AsynchronousFtpUpLoader
    {  
        // Command line arguments are two strings:
        // 1. The url that is the name of the file being uploaded to the server.
        // 2. The name of the file on the local machine.
        //
        public static void Main(string[] args)
        {
            // Create a Uri instance with the specified URI string.
            // If the URI is not correctly formed, the Uri constructor
            // will throw an exception.
            ManualResetEvent waitObject;

            Uri target = new Uri (args[0]);
            string fileName = args[1];
            FtpState state = new FtpState();
            FtpWebRequest request = (FtpWebRequest)WebRequest.Create(target);
            request.Method = WebRequestMethods.Ftp.UploadFile;

            // This example uses anonymous logon.
            // The request is anonymous by default; the credential does not have to be specified. 
            // The example specifies the credential only to
            // control how actions are logged on the server.

            request.Credentials = new NetworkCredential ("anonymous","janeDoe@contoso.com");

            // Store the request in the object that we pass into the
            // asynchronous operations.
            state.Request = request;
            state.FileName = fileName;

            // Get the event to wait on.
            waitObject = state.OperationComplete;

            // Asynchronously get the stream for the file contents.
            request.BeginGetRequestStream(
                new AsyncCallback (EndGetStreamCallback), 
                state
            );

            // Block the current thread until all operations are complete.
            waitObject.WaitOne();

            // The operations either completed or threw an exception.
            if (state.OperationException != null)
            {
                throw state.OperationException;
            }
            else
            {
                Console.WriteLine("The operation completed - {0}", state.StatusDescription);
            }
        }
        private static void EndGetStreamCallback(IAsyncResult ar)
        {
            FtpState state = (FtpState) ar.AsyncState;

            Stream requestStream = null;
            // End the asynchronous call to get the request stream.
            try
            {
                requestStream = state.Request.EndGetRequestStream(ar);
                // Copy the file contents to the request stream.
                const int bufferLength = 2048;
                byte[] buffer = new byte[bufferLength];
                int count = 0;
                int readBytes = 0;
                FileStream stream = File.OpenRead(state.FileName);
                do
                {
                    readBytes = stream.Read(buffer, 0, bufferLength);
                    requestStream.Write(buffer, 0, readBytes);
                    count += readBytes;
                }
                while (readBytes != 0);
                Console.WriteLine ("Writing {0} bytes to the stream.", count);
                // IMPORTANT: Close the request stream before sending the request.
                requestStream.Close();
                // Asynchronously get the response to the upload request.
                state.Request.BeginGetResponse(
                    new AsyncCallback (EndGetResponseCallback), 
                    state
                );
            } 
            // Return exceptions to the main application thread.
            catch (Exception e)
            {
                Console.WriteLine("Could not get the request stream.");
                state.OperationException = e;
                state.OperationComplete.Set();
                return;
            }

        }

        // The EndGetResponseCallback method  
        // completes a call to BeginGetResponse.
        private static void EndGetResponseCallback(IAsyncResult ar)
        {
            FtpState state = (FtpState) ar.AsyncState;
            FtpWebResponse response = null;
            try 
            {
                response = (FtpWebResponse) state.Request.EndGetResponse(ar);
                response.Close();
                state.StatusDescription = response.StatusDescription;
                // Signal the main application thread that 
                // the operation is complete.
                state.OperationComplete.Set();
            }
            // Return exceptions to the main application thread.
            catch (Exception e)
            {
                Console.WriteLine ("Error getting response.");
                state.OperationException = e;
                state.OperationComplete.Set();
            }
        }
    }
}

注意:该问题并非在所有环境中都可重现,但在我要部署到的环境(Pivotal Cloud Foundry)上是一致的。

故障排除,我推测在计算签名时,有一个环境因素会使boto3感到困惑。 对于boto3-Pivotal Cloud Foundry交互,是否有什么想法会导致此异常?

如果可能的话,您可以指导我了解boto3如何验证签名的方向。

例外

import sys
import boto3
import botocore
from boto3.session import Session, Config

def transfer(bucket, access_key, secret_key, file_name, byte_range):
    src_s3 = boto3.client('s3', 
                        verify=False,
                        aws_access_key_id = access_key,
                        aws_secret_access_key = secret_key,
                        config = Config(signature_version='s3v4')
                        )

    try:
        print('Now will start reading using byte_range:', byte_range)
        part_object = src_s3.get_object(Bucket=bucket, Key=file_name, Range=byte_range)
        print('Done reading byte_range:', byte_range)
    except Exception as e:
        print('Failed reading byte_range:', byte_range)
        print(e)
        raise



if __name__ == "__main__":
    bucket = sys.argv[1]
    access_key = sys.argv[2]
    secret_key = sys.argv[3]
    file_name = sys.argv[4]
    byte_range = sys.argv[5]

    transfer(bucket, access_key, secret_key, file_name, byte_range)

0 个答案:

没有答案