Question

我想使用AWS Spot实例来训练神经网络。为了防止终点实例终止时丢失模型，我计划创建EBS卷的快照，创建一个新卷并将其附加到保留的实例。如何使用python＆amp; amp安装或使EBS卷可用; boto3。

这些是Linux上用于make the volume available的步骤，但我希望自动化该过程，以便每次都不需要SSH到实例中。这是我用来附加卷的代码 -

import boto3
ec2 = boto3.resource('ec2')

spot = ec2.Instance('i-9a8f5082')
res = ec2.Instance('i-86e65a13')

snapshot = ec2.create_snapshot(VolumeId="vol-5315f7db", Description="testing spot instances")
volume = ec2.create_volume(SnapshotId=snapshot.id, AvailabilityZone='us-west-2a')
res.attach_volume(VolumeId="vol-5315f7db", Device='/dev/sdy')
snapshot.delete()

Answer 1

您必须在操作系统中执行这些步骤。您无法通过AWS API（Boto3）执行这些步骤。最好的办法是编写脚本，然后通过Boto3以某种方式启动脚本，可能使用AWS SSM服务。

Answer 2

远程发送和执行ssh脚本有什么问题？假设您使用的是ubuntu，即

ssh -i  your.pem ubuntu@ec2_name_or_ip  'sudo bash -s' < mount_script.sh

如果您将标记附加到这些资源，您可以稍后使用boto3通过通用标记名称查询资源，而不是绑定到特定的静态ID。

Answer 3

您需要在实例上运行mount命令。两种方式。一个是带有ssh连接的发送命令，如@mootmoot所写。另一个是带有AWS SSM服务的发送命令，如@Mark B所写。以下是详细的SSM解决方案示例，您可以忽略不必要的部分：

使用AWS SSM向实例发送bash命令：

# Amazon EC2 Systems Manager requires
# 1. An IAM role for EC2 instances that will process commands. There should be a system manager role and the instance should use this role ! (Did it while creation instance)
# 2. And a separate role for users executing commands. Aws IAM user that has access and secret keys should have ssm permission. (i.e. AmazonSSMFullAccess)
# http://docs.aws.amazon.com/systems-manager/latest/userguide/sysman-configuring-access-policies.html
def execute_commands_on_linux_instances(commands, instance_ids):
    client = boto3.client('ssm', **conn_args) # Need your credentials here

    all_ssm_enabled_instances, ssm_enabled_instances, not_worked_instances = [],[],[]
    not_worked_instances = instance_ids.copy()
    all_ssm_enabled_instances = list()
    outputs = list({})
    not_executed = list()

    # Select only the Instances that have an active ssm agent.
    if len(client.describe_instance_information()['InstanceInformationList']) > 0:
        resp = client.describe_instance_information(MaxResults=20)['InstanceInformationList']
        for ins in resp:
            all_ssm_enabled_instances.append(ins['InstanceId'])
        ssm_enabled_instances = list(set(all_ssm_enabled_instances).intersection(instance_ids))
        not_worked_instances = list(set(instance_ids).difference(all_ssm_enabled_instances))


        # Now, send the command !
        resp = client.send_command(
        DocumentName="AWS-RunShellScript",
        Parameters={'commands': [commands]},
        InstanceIds=ssm_enabled_instances,
        )

        # get the command id generated by the send_command
        com_id = resp['Command']['CommandId']

        # Wait until all the commands status are out of Pending and InProgress
        list_comm = client.list_commands( CommandId=com_id)
        while True:
            list_comm = client.list_commands( CommandId=com_id)
            if (list_comm['Commands'][0]['Status'] == 'Pending'or list_comm['Commands'][0]['Status'] == 'InProgress'):
                continue
            else:
                # Commands on all Instances were executed
                break

        # Get the responses the instances gave to this command. (stdoutput and stderror)
        # Althoug the command could arrive to instance, if it couldn't be executed by the instance (response -1) it will ignore.
        for i in ssm_enabled_instances:
            resp2 = client.get_command_invocation(CommandId=com_id, InstanceId=i)
            if resp2['ResponseCode'] == -1:
                not_executed.append(i)
            else:
                outputs.append({'ins_id': i, 'stdout': resp2['StandardOutputContent'],
                            'stderr': resp2['StandardErrorContent']})

        # Remove the instance that couldn't execute the command ever, add it to not_worked_instances
        ssm_enabled_instances = list(set(ssm_enabled_instances).difference(not_executed))
        not_worked_instances.extend(not_executed)

        return ssm_enabled_instances, not_worked_instances, outputs
    else:
        print("There is no any available instance that has a worked SSM service!")
        return ssm_enabled_instances,  not_worked_instances, outputs

使用必需的具有所需策略的角色的IAM实例配置文件创建实例。作为此实例创建的结果，实例已运行SSM代理：

def create_ec2_instance(node_type):
    # define userdata to be run at instance launch

    userdata = """#cloud-config

    runcmd:
     - cd /tmp
     - sudo yum install -y https://s3.amazonaws.com/ec2-downloads-windows/SSMAgent/latest/linux_amd64/amazon-ssm-agent.rpm
    """

    ec2_r = boto3.resource('ec2', **conn_args)

    rolename = "amazonec2ssmrole"
    i_pro_name = "ins_pro_for_ssm"

    # Create an iam instance profile and add required role to this instance profile.
    # Create a role and attach a policy to it if not exist.
    # Instances will have this role to build ssm (ec2 systems manager) connection.
    iam = boto3.resource('iam', **conn_args)

    try:
        response= iam.meta.client.get_instance_profile(InstanceProfileName=i_pro_name)
    except:
        iam.create_instance_profile(InstanceProfileName=i_pro_name)
    try:
        response = iam.meta.client.get_role(RoleName=rolename)
    except:
        iam.create_role(
                    AssumeRolePolicyDocument='{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"Service":["ec2.amazonaws.com"]},"Action":["sts:AssumeRole"]}]}',
                    RoleName=rolename)
        role = iam.Role(rolename)
        role.attach_policy(PolicyArn='arn:aws:iam::aws:policy/service-role/AmazonEC2RoleforSSM')
        iam.meta.client.add_role_to_instance_profile(InstanceProfileName=i_pro_name, RoleName=rolename)

    iam_ins_profile = {'Name': i_pro_name}

    if node_type == "Medium":
        instance = ec2_r.create_instances(
            ImageId='ami-aa5ebdd2',
            MinCount=1,
            MaxCount=1,
            UserData=userdata,
            InstanceType='t2.medium',
            KeyName=key_pair_name,
            IamInstanceProfile=iam_ins_profile,
            BlockDeviceMappings=[{"DeviceName": "/dev/xvda", "Ebs": {"VolumeSize": 20}}])
    elif node_type == "Micro":
        instance = ec2_r.create_instances(
            ImageId='ami-aa5ebdd2',
            MinCount=1,
            MaxCount=1,
            UserData=userdata,
            InstanceType='t2.micro',
            KeyName=key_pair_name,
            IamInstanceProfile=iam_ins_profile,
            BlockDeviceMappings=[{"DeviceName": "/dev/xvda", "Ebs": {"VolumeSize": 10}}])
    else:
        print("Node Type Error")
        return -1

    # Wait for the instance state, default --> one wait is 15 seconds, 40 attempts
    print('Waiting for instance {0} to switch to running state'.format(instance[0].id))
    waiter = ec2_r.meta.client.get_waiter('instance_running')
    waiter.wait(InstanceIds=[instance[0].id])
    instance[0].reload()
    print('Instance is running, public IP: {0}'.format(instance[0].public_ip_address))

    return instance[0].id

不要忘记给予ssm许可。（即AmazonSSMFullAccess）给具有访问权限和密钥的Aws IAM用户。

顺便说一句，conn_args可以定义如下：

 conn_args = {
        'aws_access_key_id': Your_Access_Key,
        'aws_secret_access_key': Your_Secret_Key,
        'region_name': 'us-west-2'
    }

python - 使用boto3安装EBS卷

3 个答案: