我在Confluent社区平台中使用Kafka Connect来保持MySQL数据库同步。源和接收器是MySQL数据库。没用。
MySQL Sync purpose 我的连接器源配置为：
HWND hMyWnd = NULL;
HANDLE hThread = NULL;
bool stopThread = false;
int readString(SOCKET sock, char *buf, int buflen, string &str)
{
str.clear();
char ch, *pbuf = buf;
int len = 0, bytesReceived;
do {
if (stopThread)
return -2;
bytesReceived = recv(sock, &ch, 1, 0);
if (bytesReceived == -1) {
if ((WSAGetLastError() != WSAETIMEDOUT) || !str.empty()) {
//cerr << "Can't read from socket, Err#" << WSAGetLastError() << endl;
return -1;
}
return 1;
}
if (bytesReceived == 0) {
//cerr << "Socket disconnected by server" << endl;
return 0;
}
if (ch == '\0')
break;
*pbuf++ = ch;
len += bytesReceived;
if (len == buflen) {
str.append(buf, len);
pbuf = buf;
len = 0;
}
}
while (true);
if (len > 0)
str.append(buf, len);
return 1;
}
int sendString(SOCKET sock, const string &str)
{
const char *pbuf = str.c_str();
int len = str.length() + 1, bytesSent;
do {
if (stopThread)
return -2;
bytesSent = send(sock, pbuf, len, 0);
if (bytesSent == -1) {
//cerr << "Can't send to socket, Err#" << WSAGetLastError() << endl;
return -1;
}
pbuf += bytesSent;
len -= bytesSent;
}
while (len > 0);
return 1;
}
//Things to compare
const char* CreateAccount = "create"; //Server tells client to make IG account
const char* CheckStatus = "check"; //Client tells server if account is running or not
const char* Info = "info"; //Client sends Username and Password of account to server
const char* Run = "run"; //Tells client to start running the account
const char* Kill = "kill"; //Kills program on client for around a month
const char* Settings = "settings"; //Server sets settings for account to run on
//Things to send
const string TryAgain = "#13 Not a valid input, either type [check] to check if the account is running, type [create] to create new account, or type [info] for account information\n";
const string accInfoSuccess = "#777 You have successfully entered the information for the account (^.^)\n";
const string settingsInfoSuccess = "#777 You have successfully set the settings for this account (^.^)\n";
const string accInfoProblem = "#13 There was a problem in either saving the information to this account or the account creation program. Type [create] and try again (T.T)\n";
const string settingsInfoProblem = "#13 There was a problem in saving the account settings. Type [settings] and try again (T.T)\n";
const string accRun = "#777 The account is currently running! ~(^.^)~\n";
const string accNoRun = "#13 Sorry the account isn't running, type [run] to run the account (O.o)\n";
const string accNoInfo = "#13 There is no info for an account that can be run. Type [create] to make information (\".\")\n";
const string accRunErr = "#13 There is a problem with opening the main bot program, try again. If this problem persists, there is an issue clientside V(T.T)V\n";
DWORD WINAPI ThreadProc(LPVOID lpParameter)
{
string ipAddress = "10.0.0.201"; //IP address of my computer on local network
int port = 13777;
WSAData data;
WORD ver = MAKEWORD(2, 2);
int wsResult = WSAStartup(ver, &data);
if (wsResult != 0) {
//cerr << "Can't start Winsock, Err#" << wsResult << endl;
return 0;
}
sockaddr_in hint = {};
hint.sin_family = AF_INET;
hint.sin_port = htons(port);
inet_pton(AF_INET, ipAddress.c_str(), &hint.sin_addr);
SOCKET sock = INVALID_SOCKET;
char buf[1024]; //Where message from server will be stored
string str;
while (!stopThread) {
//attempt to connect to server
sock = socket(AF_INET, SOCK_STREAM, 0);
if (sock == INVALID_SOCKET) {
//cerr << "Can't create socket, Err#" << WSAGetLastError() << endl;
break;
}
//BOOL on = TRUE;
//setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, (char*)&on, sizeof(on));
//u_long block = 0;
//ioctlsocket(sock, FIONBIO, &block);
DWORD timeout = 500;
setsockopt(sock, SOL_SOCKET, SO_RCVTIMEO, (char*)&timeout, sizeof(timeout));
//setsockopt(sock, SOL_SOCKET, SO_SNDTIMEO, (char*)&timeout, sizeof(timeout));
wsResult = connect(sock, (sockaddr*)&hint, sizeof(hint));
if (wsResult != SOCKET_ERROR) {
do {
wsResult = readString(sock, buf, sizeof(buf), str);
if (wsResult <= 0)
break;
if (!str.empty()) {
//Check to see if it equals one of the strings above
if (str == CreateAccount) {
//Run program to create account
if (accountCreation(sock) == 0) {
wsResult = sendString(sock, accInfoSuccess);
}
else {
wsResult = sendString(sock, accInfoProblem);
}
}
else if (str == Settings) {
if (settingsCreation(sock) == 0) {
wsResult = sendString(sock, settingsInfoSuccess);
}
else {
wsResult = sendString(sock, settingsInfoProblem);
}
}
else if (str == CheckStatus) {
//Check to see if program that runs account is running
int accountSuccess = isRunning();
if (accountSuccess == 0) {
wsResult = sendString(sock, accRun);
}
else if (accountSuccess == 1){
wsResult = sendString(sock, accNoRun);
}
else {
wsResult = sendString(sock, accNoInfo);
}
}
else if (str == Info) {
//Read text file containing account info and send to server
if (checkInfo(sock) != 0) {
wsResult = sendString(sock, accNoInfo);
}
}
else if (str == Run) {
//Runs the account running program
if (runProg() == 0) {
wsResult = sendString(sock, accRun);
}
else {
wsResult = sendString(sock, accRunErr);
}
}
else if (str == Kill) {
//Kills this program
PostMessage(hMyWnd, WM_CLOSE, 0, 0);
stopThread = true;
break;
}
else {
//Send back to server that the wrong thing was inputted
wsResult = sendString(sock, TryAgain);
}
}
else {
//Check to make sure bot is running
if (isRunning() == 1) {
//runProg();
}
}
}
while (!stopThread);
}
closesocket(sock);
sock = INVALID_SOCKET;
if (!stopThread)
Sleep(5000);
}
WSACleanup();
return 0;
}
LRESULT CALLBACK WindowProc(HWND hWnd, UINT message, WPARAM wParam, LPARAM lParam) {
switch (message) {
case WM_CREATE: {
hThread = CreateThread(NULL, 0, ThreadProc, NULL, 0, NULL);
return 0;
case WM_POWERBROADCAST:
DestroyWindow(hWnd);
return 0;
case WM_DESTROY:
if (hThread) {
stopThread = true;
WaitForSingleObject(hThread, INFINITE);
CloseHandle(hThread);
hThread = NULL;
}
PostQuitMessage(0);
return 0;
case WM_CLOSE:
DestroyWindow(hWnd);
return 0;
}
return DefWindowProc(hWnd, message, wParam, lParam);
}
int WINAPI WinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance, LPSTR lpCmdLine, int nCmdShow)
{
//Change the current directory of the program back to the appropriate folder
wchar_t* UserProf;
if (SHGetKnownFolderPath(FOLDERID_LocalAppData, 0, NULL, &UserProf) == S_OK) {
wstring wstr = wstring(UserProf) + L"\\UserUpdates";
//wcout << wstr << endl;
SetCurrentDirectoryW(wstr.c_str());
CoTaskMemFree(UserProf);
}
WNDCLASSW WindowClass{CS_NOCLOSE, WindowProc, 0, 0, hInstance, NULL, LoadCursor(nullptr, IDC_ARROW), NULL, NULL, L"chakra"};
if (!RegisterClassW(&WindowClass)) {
//cerr << "Can't register window class, Err#" << GetLastError() << endl;
return 0;
}
hMyWnd = CreateWindowW(L"chakra", L"star", WS_POPUP, 0, 0, 10, 10, NULL, NULL, hInstance, NULL);
if (!hMyWnd) {
//cerr << "Can't create window, Err#" << GetLastError() << endl;
return 0;
}
ShowWindow(hMyWnd, SW_HIDE);
MSG Msg;
while (GetMessage(&Msg, NULL, 0, 0)) {
TranslateMessage(&Msg);
DispatchMessage(&Msg);
}
return 0;
}
但是json中的消息仍然像这样：
{
"name":"br-auths-3910472223-source",
"config":{
"connector.class": "io.debezium.connector.mysql.MySqlConnector",
"tasks.max": "1",
"database.hostname": "localhost",
"database.port": "3306",
"database.user": "root",
"database.password": "br123456",
"database.useLegacyDatetimeCode": "false",
"database.server.id": "184",
"database.server.name": "local3910472223",
"database.whitelist":"br_auths",
"database.history.kafka.bootstrap.servers": "localhost:9092",
"database.history.kafka.topic": "schema-changes.br-auths.local3910472223" ,
"table.blacklist": "br_auths.__migrationversions,br_auths.auths_service_apps",
"include.schema.changes": "true",
"transforms": "route,TimestampConverter",
"transforms.TimestampConverter.type": "org.apache.kafka.connect.transforms.TimestampConverter$Value",
"transforms.TimestampConverter.target.type": "string",
"transforms.TimestampConverter.field": "payload.after.ctime",
"transforms.TimestampConverter.format": "yyyy-MM-dd HH:mm:ss",
"transforms.route.type": "org.apache.kafka.connect.transforms.RegexRouter",
"transforms.route.regex": "([^.]+)\\.([^.]+)\\.([^.]+)",
"transforms.route.replacement": "$2__$3"
}
}
ctime，utime列被序列化为一个大整数，当JDBC Sink Connector尝试插入MySQL datetime列时，它抛出了格式异常，它需要'YYYY-MM-DD HH：mm：ss'字符串格式。我的TimestampConverter转换无效。
有什么办法可以正确地输入和输出datetime列？