我正在使用userDefinedRouting和与网络安全组关联的现有子网和路由表来设置AKS群集。这是我的代码段。
provider "azurerm" {
version = "~> 2.25"
features {}
}
data "azurerm_resource_group" "aks" {
name = var.resource_group
}
#fetch existing subnet
data "azurerm_subnet" "aks" {
name = var.subnetname
virtual_network_name = var.virtual_network_name
resource_group_name = var.vnet_resource_group
}
resource "azurerm_network_interface" "k8svmnic" {
name = "k8svmnic"
resource_group_name = data.azurerm_resource_group.aks.name
location = data.azurerm_resource_group.aks.location
ip_configuration {
name = "internal"
subnet_id = data.azurerm_subnet.aks.id
private_ip_address_allocation = "Static"
private_ip_address = var.k8svmip #"10.9.56.10"
}
}
resource "azurerm_availability_set" "k8svmavset" {
name = "k8svmavset"
location = data.azurerm_resource_group.aks.location
resource_group_name = data.azurerm_resource_group.aks.name
platform_fault_domain_count = 3
platform_update_domain_count = 3
managed = true
}
resource "azurerm_network_security_group" "k8svmnsg" {
name = "k8vm-nsg"
resource_group_name = data.azurerm_resource_group.aks.name
location = data.azurerm_resource_group.aks.location
security_rule {
name = "allow_kube_tls"
protocol = "Tcp"
priority = 100
direction = "Inbound"
access = "Allow"
source_address_prefix = "VirtualNetwork"
destination_address_prefix = "*"
source_port_range = "*"
#destination_port_range = "443"
destination_port_ranges = ["443"]
description = "Allow kube-apiserver (tls) traffic to master"
}
security_rule {
name = "allow_ssh"
protocol = "Tcp"
priority = 101
direction = "Inbound"
access = "Allow"
source_address_prefix = "*"
destination_address_prefix = "*"
source_port_range = "*"
#destination_port_range = "22"
destination_port_ranges = ["22"]
description = "Allow SSH traffic to master"
}
}
resource "azurerm_network_interface_security_group_association" "k8svmnicnsg" {
network_interface_id = azurerm_network_interface.k8svmnic.id
network_security_group_id = azurerm_network_security_group.k8svmnsg.id
}
resource "azurerm_linux_virtual_machine" "k8svm" {
name = "k8svm"
resource_group_name = data.azurerm_resource_group.aks.name
location = data.azurerm_resource_group.aks.location
size = "Standard_D3_v2"
admin_username = var.admin_username
disable_password_authentication = true
availability_set_id = azurerm_availability_set.k8svmavset.id
network_interface_ids = [
azurerm_network_interface.k8svmnic.id,
]
admin_ssh_key {
username = var.admin_username
public_key = var.ssh_key
}
os_disk {
caching = "ReadWrite"
storage_account_type = "Standard_LRS"
disk_size_gb = 30
}
source_image_reference {
publisher = "microsoft-aks"
offer = "aks"
sku = "aks-engine-ubuntu-1804-202007"
version = "2020.07.24"
}
}
resource "azurerm_managed_disk" "k8svm-disk" {
name = "${azurerm_linux_virtual_machine.k8svm.name}-disk"
location = data.azurerm_resource_group.aks.location
resource_group_name = data.azurerm_resource_group.aks.name
storage_account_type = "Standard_LRS"
create_option = "Empty"
disk_size_gb = 512
}
resource "azurerm_virtual_machine_data_disk_attachment" "k8svm-disk-attachment" {
managed_disk_id = azurerm_managed_disk.k8svm-disk.id
virtual_machine_id = azurerm_linux_virtual_machine.k8svm.id
lun = 5
caching = "ReadWrite"
}
resource "azurerm_public_ip" "aks" {
name = "akspip"
resource_group_name = data.azurerm_resource_group.aks.name
location = data.azurerm_resource_group.aks.location
allocation_method = "Static"
sku = "Standard"
depends_on = [azurerm_virtual_machine_data_disk_attachment.k8svm-disk-attachment]
}
resource "azurerm_route_table" "aks"{
name = "aks" #var.subnetname
resource_group_name = data.azurerm_resource_group.aks.name
location = data.azurerm_resource_group.aks.location
disable_bgp_route_propagation = false
route {
name = "default_route"
address_prefix = "0.0.0.0/0"
next_hop_type = "VirtualAppliance"
next_hop_in_ip_address = var.k8svmip
}
route {
name = var.route_name
address_prefix = var.route_address_prefix
next_hop_type = var.route_next_hop_type
}
}
resource "azurerm_subnet_route_table_association" "aks" {
subnet_id = data.azurerm_subnet.aks.id
route_table_id = azurerm_route_table.aks.id
}
resource "azurerm_subnet_network_security_group_association" "aks" {
subnet_id = data.azurerm_subnet.aks.id
network_security_group_id = var.network_security_group
}
resource "null_resource" "previous" {}
resource "time_sleep" "wait_90_seconds" {
depends_on = [null_resource.previous]
create_duration = "90s"
}
# This resource will create (at least) 30 seconds after null_resource.previous
resource "null_resource" "next" {
depends_on = [time_sleep.wait_90_seconds]
}
resource "azurerm_kubernetes_cluster" "aks" {
name = data.azurerm_resource_group.aks.name
resource_group_name = data.azurerm_resource_group.aks.name
location = data.azurerm_resource_group.aks.location
dns_prefix = "akstfelk" #The dns_prefix must contain between 3 and 45 characters, and can contain only letters, numbers, and hyphens. It must start with a letter and must end with a letter or a number.
kubernetes_version = "1.18.8"
private_cluster_enabled = false
node_resource_group = var.node_resource_group
#api_server_authorized_ip_ranges = [] #var.api_server_authorized_ip_ranges
default_node_pool {
enable_node_public_ip = false
name = "agentpool"
node_count = var.node_count
orchestrator_version = "1.18.8"
vm_size = var.vm_size
os_disk_size_gb = var.os_disk_size_gb
vnet_subnet_id = data.azurerm_subnet.aks.id
type = "VirtualMachineScaleSets"
}
linux_profile {
admin_username = var.admin_username
ssh_key {
key_data = var.ssh_key
}
}
service_principal {
client_id = var.client_id
client_secret = var.client_secret
}
role_based_access_control {
enabled = true
}
network_profile {
network_plugin = "kubenet"
network_policy = "calico"
dns_service_ip = "172.16.1.10"
service_cidr = "172.16.0.0/16"
docker_bridge_cidr = "172.17.0.1/16"
pod_cidr = "172.40.0.0/16"
outbound_type = "userDefinedRouting"
load_balancer_sku = "Standard"
load_balancer_profile {
outbound_ip_address_ids = [ "${azurerm_public_ip.aks.id}" ]
}
# load_balancer_profile {
# managed_outbound_ip_count = 5
# #effective_outbound_ips = [ azurerm_public_ip.aks.id ]
# outbound_ip_address_ids = []
# outbound_ip_prefix_ids = []
# outbound_ports_allocated = 0
# }
}
addon_profile {
aci_connector_linux {
enabled = false
}
azure_policy {
enabled = false
}
http_application_routing {
enabled = false
}
kube_dashboard {
enabled = false
}
oms_agent {
enabled = false
}
}
depends_on = [azurerm_subnet_route_table_association.aks]
}
根据Azure文档说：“默认情况下，如果未指定公共IP，公共IP前缀或IP数量，则会在与AKS群集相同的资源组中自动创建一个公共IP。
但是在我的情况下，没有发生出站连接，因此群集设置失败。我什至创建了另一个公共IP并尝试通过Loadbalancer配置文件，但是我遇到了错误。
Error: "network_profile.0.load_balancer_profile.0.managed_outbound_ip_count": conflicts with network_profile.0.load_balancer_profile.0.outbound_ip_address_ids
如果我从脚本中删除了loadbalancer_profile，我将得到错误提示
Error: creating Managed Kubernetes Cluster "aks-tf" (Resource Group "aks-tf"): containerservice.ManagedClustersClient#CreateOrUpdate: Failure sending request: StatusCode=400 -- Original Error: Code="InvalidUserDefinedRoutingWithLoadBalancerProfile" Message="UserDefinedRouting and load balancer profile are mutually exclusive. Please refer to http://aka.ms/aks/outboundtype for more details" Target="networkProfile.loadBalancerProfile"
Kinldy帮我想念我的地方。
任何帮助将不胜感激。
答案 0 :(得分：0)
使用UserDefineRouting时，需要将network_plugin设置为azure并使用用户定义的路由器将AKS群集放入子网中，here为说明：
必须将AKS群集部署到具有以下功能的现有虚拟网络中： 先前已配置的子网。
如果将network_plugin设置为azure，则必须设置default_node_pool块中的vnet_subnet_id字段，并且不能设置pod_cidr。您可以在azurerm_kubernetes_cluster中找到此注释。
更新：
它比您想象的要复杂一些，here是它的网络体系结构以及通过CLI创建它的步骤。这种架构要求将出口流量显式发送到防火墙，网关，代理之类的设备，或者允许网络地址转换（NAT）由分配给标准负载均衡器或设备的公共IP完成。
对于出站，可以使用内部负载平衡器代替内部公共负载平衡器来代替内部流量。
此外，某些无法通过Terraform实现的步骤，例如Azure防火墙。查看步骤，并准备无法通过CLI实现的资源。