Consul-Template意外的响应代码:504

时间:2017-08-16 11:02:38

标签: consul consul-template consul-kv

我正在使用Consul-Template来呈现包含从Consul收集的数据的属性。文件正在成功呈现,但之后Consul-Template一遍又一遍地失败,直到它崩溃:

2017/08/16 10:43:04.900637 [WARN] (view) kv.list(...): Unexpected response code: 504 (retry attempt 5 after "4s")

这会持续更长的暂停,直到达到极限/它崩溃。

从日志中提取配置:

2017/08/16 10:38:02.403969 [DEBUG] (runner) final config:
{
    "Consul": {
        "Address": "consul-client.development.discovery",
        "Auth": {
            "Enabled": false,
            "Username": "",
            "Password": ""
        },
        "Retry": {
            "Attempts": 12,
            "Backoff": 250000000,
            "MaxBackoff": 60000000000,
            "Enabled": true
        },
        "SSL": {
            "CaCert": "",
            "CaPath": "",
            "Cert": "",
            "Enabled": false,
            "Key": "",
            "ServerName": "",
            "Verify": true
        },
        "Token": "",
        "Transport": {
            "DialKeepAlive": 30000000000,
            "DialTimeout": 30000000000,
            "DisableKeepAlives": false,
            "IdleConnTimeout": 90000000000,
            "MaxIdleConns": 100,
            "MaxIdleConnsPerHost": 3,
            "TLSHandshakeTimeout": 10000000000
        }
    },
    "Dedup": {
        "Enabled": false,
        "MaxStale": 2000000000,
        "Prefix": "consul-template/dedup/",
        "TTL": 15000000000
    },
    "Exec": {
        "Command": "",
        "Enabled": false,
        "Env": {
            "Blacklist": [],
            "Custom": [],
            "Pristine": false,
            "Whitelist": []
        },
        "KillSignal": 2,
        "KillTimeout": 30000000000,
        "ReloadSignal": null,
        "Splay": 0,
        "Timeout": 0
    },
    "KillSignal": 2,
    "LogLevel": "DEBUG",
    "MaxStale": 2000000000,
    "PidFile": "",
    "ReloadSignal": 1,
    "Syslog": {
        "Enabled": false,
        "Facility": "LOCAL0"
    },
    "Templates": [
        {
            "Backup": false,
            "Command": "",
            "CommandTimeout": 30000000000,
            "Contents": "",
            "Destination": "/consul-template/output/service.properties",
            "Exec": {
                "Command": "",
                "Enabled": false,
                "Env": {
                    "Blacklist": [],
                    "Custom": [],
                    "Pristine": false,
                    "Whitelist": []
                },
                "KillSignal": 2,
                "KillTimeout": 30000000000,
                "ReloadSignal": null,
                "Splay": 0,
                "Timeout": 30000000000
            },
            "Perms": 420,
            "Source": "/consul-template/data/service.properties.tpl",
            "Wait": {
                "Enabled": false,
                "Min": 0,
                "Max": 0
            },
            "LeftDelim": "",
            "RightDelim": ""
        },
        {
            "Backup": false,
            "Command": "",
            "CommandTimeout": 30000000000,
            "Contents": "",
            "Destination": "/consul-template/output/application.properties",
            "Exec": {
                "Command": "",
                "Enabled": false,
                "Env": {
                    "Blacklist": [],
                    "Custom": [],
                    "Pristine": false,
                    "Whitelist": []
                },
                "KillSignal": 2,
                "KillTimeout": 30000000000,
                "ReloadSignal": null,
                "Splay": 0,
                "Timeout": 30000000000
            },
            "Perms": 420,
            "Source": "/consul-template/data/application.properties.tpl",
            "Wait": {
                "Enabled": false,
                "Min": 0,
                "Max": 0
            },
            "LeftDelim": "",
            "RightDelim": ""
        },
        {
            "Backup": false,
            "Command": "",
            "CommandTimeout": 30000000000,
            "Contents": "",
            "Destination": "/consul-template/output/anotherMicroConsul.properties",
            "Exec": {
                "Command": "",
                "Enabled": false,
                "Env": {
                    "Blacklist": [],
                    "Custom": [],
                    "Pristine": false,
                    "Whitelist": []
                },
                "KillSignal": 2,
                "KillTimeout": 30000000000,
                "ReloadSignal": null,
                "Splay": 0,
                "Timeout": 30000000000
            },
            "Perms": 420,
            "Source": "/consul-template/data/anotherMicroConsul.properties.tpl",
            "Wait": {
                "Enabled": false,
                "Min": 0,
                "Max": 0
            },
            "LeftDelim": "",
            "RightDelim": ""
        },
        {
            "Backup": false,
            "Command": "",
            "CommandTimeout": 30000000000,
            "Contents": "",
            "Destination": "/consul-template/output/master.properties",
            "Exec": {
                "Command": "",
                "Enabled": false,
                "Env": {
                    "Blacklist": [],
                    "Custom": [],
                    "Pristine": false,
                    "Whitelist": []
                },
                "KillSignal": 2,
                "KillTimeout": 30000000000,
                "ReloadSignal": null,
                "Splay": 0,
                "Timeout": 30000000000
            },
            "Perms": 420,
            "Source": "/consul-template/data/master.properties.tpl",
            "Wait": {
                "Enabled": false,
                "Min": 0,
                "Max": 0
            },
            "LeftDelim": "",
            "RightDelim": ""
        },
        {
            "Backup": false,
            "Command": "",
            "CommandTimeout": 30000000000,
            "Contents": "",
            "Destination": "/consul-template/output/microConsul.properties",
            "Exec": {
                "Command": "",
                "Enabled": false,
                "Env": {
                    "Blacklist": [],
                    "Custom": [],
                    "Pristine": false,
                    "Whitelist": []
                },
                "KillSignal": 2,
                "KillTimeout": 30000000000,
                "ReloadSignal": null,
                "Splay": 0,
                "Timeout": 30000000000
            },
            "Perms": 420,
            "Source": "/consul-template/data/microConsul.properties.tpl",
            "Wait": {
                "Enabled": false,
                "Min": 0,
                "Max": 0
            },
            "LeftDelim": "",
            "RightDelim": ""
        }
    ],
    "Vault": {
        "Address": "",
        "Enabled": false,
        "Grace": 15000000000,
        "RenewToken": true,
        "Retry": {
            "Attempts": 12,
            "Backoff": 250000000,
            "MaxBackoff": 60000000000,
            "Enabled": true
        },
        "SSL": {
            "CaCert": "",
            "CaPath": "",
            "Cert": "",
            "Enabled": true,
            "Key": "",
            "ServerName": "",
            "Verify": true
        },
        "Transport": {
            "DialKeepAlive": 30000000000,
            "DialTimeout": 30000000000,
            "DisableKeepAlives": false,
            "IdleConnTimeout": 90000000000,
            "MaxIdleConns": 100,
            "MaxIdleConnsPerHost": 3,
            "TLSHandshakeTimeout": 10000000000
        },
        "UnwrapToken": false
    },
    "Wait": {
        "Enabled": false,
        "Min": 0,
        "Max": 0
    }
}

通过代码筛选此错误消息似乎来自session.go中的Renew

我是否应该在/session/list下的Consul会面?我有多个Consul-Template容器在运行(每个主机上有一个) - 我应该看几个会话吗?现在我什么也看不见......

curl localhost:8500/v1/session/list
[]

1 个答案:

答案 0 :(得分:2)

原因:您的领事模板和领事之间有代理或负载均衡器。

解决方法:将空闲连接的负载均衡器或代理超时增加到60秒以上(例如:65秒或120秒),因为在此之后,keepalive将启动并保持连接处于活动状态:)

<强>详情

  • HTTP 504是一个网关超时,意味着连接已被“切断”或服务无响应但我会假设您的领事群集仍在工作,因此它是由于中间的一个活动网络设备(在联系之间)模板和领事)
  • Consul-template打开持久连接以监听更改。因此,如果连接被负载均衡器静默杀死,则最终会出现这504个错误。
  • 这些消息是良性的,因为consul-template重新连接正常。但是它们很烦人,会一遍又一遍地用相同的警告填充你的日志和磁盘空间; - )
  • 如果在AWS中,将ELB空闲超时设置为65秒,这些消息将消失