Question

我尝试了多种方法来使用Boto3通过HTTP代理将文件上传到S3，但是没有成功。我注意到执行此操作的方法随着时间的推移而发生了变化，但是有很多可能性，但是在所有实验之后，对我的代理服务器访问的唯一打击是对元数据安全凭据的请求：

[centos@ip-172-31-4-54 ~]$ sudo tail -f /var/log/squid/access.log
1568816277.161      1 172.31.10.209 TCP_MISS/404 694 GET http://169.254.169.254/latest/meta-data/iam/security-credentials/ - HIER_DIRECT/169.254.169.254 text/html

否则，我似乎无法通过代理发出实际的upload_file请求。在OSX和CentOS7上已经尝试过。我知道这个线程https://github.com/boto/boto3/issues/338，但状态对我来说还不清楚。任何提示有人吗？

Python3.6 & pip list:
argh (0.26.2)
boto3 (1.9.230)
botocore (1.12.230)
docutils (0.15.2)
jmespath (0.9.4)
pathtools (0.1.2)
pip (9.0.3)
python-dateutil (2.8.0)
PyYAML (5.1.2)
s3transfer (0.2.1)
setuptools (39.2.0)
six (1.12.0)
urllib3 (1.25.3)
watchdog (0.9.0)

我的代码使用客户端和资源方法，

import os
import sys
import threading

# Tried the monkey code method
def _get_proxies(self, url):
    return {'http': 'http://172.31.4.54:80/'}

# Code lifted from https://boto3.amazonaws.com/v1/documentation/api/latest/guide/s3-uploading-files.html
def upload_file_cli(file_name, bucket, object_name=None):
    """Upload a file to an S3 bucket
    :param file_name: File to upload
    :param bucket: Bucket to upload to
    :param object_name: S3 object name. If not specified then file_name is used
    :return: True if file was uploaded, else False
    """
    import botocore.endpoint
    botocore.endpoint.EndpointCreator._get_proxies = _get_proxies
    import boto3
    from botocore.config import Config
    from boto3.s3.transfer import TransferConfig

    #https://code-examples.net/en/q/1feddac
    os.environ['HTTP_PROXY'] = 'http://172.31.4.54:80'
    os.environ['NO_PROXY'] = '169.254.169.254'

    s3 = boto3.client('s3',region_name="eu-west-2",aws_access_key_id="",aws_secret_access_key="",config=Config(proxies={'http': '192.168.150.252:80'}))
    #s3 = boto3.client('s3',region_name="eu-west-2",aws_access_key_id="",aws_secret_access_key="")
    config = TransferConfig(multipart_threshold=1024 * 25,max_concurrency=10,multipart_chunksize=1024 * 25, use_threads=True)

    # If S3 object_name was not specified, use file_name
    if object_name is None:
        object_name = file_name.lstrip("/")

    # Upload the file
    try:
        #response = s3.upload_file(file_name, bucket, object_name,ExtraArgs={'ServerSideEncryption': 'AES256','StorageClass': 'STANDARD_IA','ContentType': 'text/pdf'},Config=config)
        response = s3.upload_file(file_name, bucket, object_name,ExtraArgs={'ServerSideEncryption': 'AES256','StorageClass': 'STANDARD_IA','ContentType': 'text/pdf'})
    except Exception as e:
        print("Error:", e)
        return False
    return True

def upload_file_res(file_name, bucket, object_name=None):
    """Upload a file to an S3 bucket
    :param file_name: File to upload
    :param bucket: Bucket to upload to
    :param object_name: S3 object name. If not specified then file_name is used
    :return: True if file was uploaded, else False
    """
    #https://code-examples.net/en/q/1feddac
    os.environ['HTTP_PROXY'] = 'http://172.31.4.54:80'
    os.environ['NO_PROXY'] = '169.254.169.254'

    import botocore.endpoint
    botocore.endpoint.EndpointCreator._get_proxies = _get_proxies
    import boto3
    from botocore.config import Config
    from boto3.s3.transfer import TransferConfig

    s3_res = boto3.resource('s3',region_name="eu-west-2",aws_access_key_id="",aws_secret_access_key="", config=Config(proxies={'http': '172.31.4.54:80'}))
    #s3_res = boto3.resource('s3',region_name="eu-west-2",aws_access_key_id="",aws_secret_access_key="")
    config = TransferConfig(multipart_threshold=1024 * 25,max_concurrency=10,multipart_chunksize=1024 * 25, use_threads=True)

   # If S3 object_name was not specified, use file_name
    if object_name is None:
        object_name = file_name.lstrip("/")

    # Upload the file
    try:
        #response = s3_res.meta.client.upload_file(file_name, bucket, object_name,ExtraArgs={'ServerSideEncryption': 'AES256','StorageClass': 'STANDARD_IA','ContentType': 'text/pdf'},Config=config)
        response = s3_res.meta.client.upload_file(file_name, bucket, object_name,ExtraArgs={'ServerSideEncryption': 'AES256','StorageClass': 'STANDARD_IA','ContentType': 'text/pdf'})
    except Exception as e:
        print("Error:", e)
        return False
    return True

if __name__ == '__main__':
    BUCKET_NAME = "datatestlandingzone"
    upload_file_cli("/home/centos/test.txt",BUCKET_NAME)
    upload_file_res("/home/centos/test.txt",BUCKET_NAME)

通过代理Boto3 S3 upload_file

0 个答案: