Question

实际上我正在尝试将经过训练的数据从系统1发送到系统2，这样我就可以在系统2中进行KNN分类。但是我觉得很难发送训练好的数据，因为它非常大。有没有办法通过套接字将庞大的数据从一个系统发送到另一个系统。

系统1

import sys
import time
import pickle
from sklearn.datasets import load_files
from sklearn.neighbors import KNeighborsClassifier
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfTransformer
from socket import socket, gethostbyname, AF_INET, SOCK_DGRAM

PORT_NUMBER = 5000
hostName = gethostbyname('0.0.0.0')
mySocket = socket( AF_INET, SOCK_DGRAM )
mySocket.bind( (hostName, PORT_NUMBER) )
print ("Test server listening on port {0}".format(PORT_NUMBER))

(data,addr) = mySocket.recvfrom(15)
print data
mySocket.sendto("Connected...", addr)

(data,addr) = mySocket.recvfrom(20000000)
msg=pickle.loads(data)
twenty_train=msg

mySocket.sendto("one", addr)

(data,addr) = mySocket.recvfrom(300000000)
ms=pickle.loads(data)
X_train_tfidf=ms

knn=KNeighborsClassifier(n_neighbors=3)
clf = knn.fit(X_train_tfidf, twenty_train)

f=open(sys.argv[1],'r')
g=f.read()
ans = g.strip('\n')
if ans.endswith(' '):
    ans = ans.rstrip(' ')
docs_new = [ans]

mySocket.sendto(ans, addr)

(data,addr) = mySocket.recvfrom(1000000)
msg2=pickle.loads(data)
X_new_tfidf=msg2

mySocket.sendto("two", addr)

predicted = clf.predict(X_new_tfidf)
(data,addr) = mySocket.recvfrom(100000)
msg3=pickle.loads(data)
names = msg3

for doc, category in zip(docs_new, predicted):
    print('%r => %s' % (doc, names[category]))

sys.exit()

系统2

import sys
import pickle
import time
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfTransformer
from sklearn.datasets import load_files
from sklearn.neighbors import KNeighborsClassifier
from socket import socket, AF_INET, SOCK_DGRAM
SERVER_IP   = '10.0.8.132'
PORT_NUMBER = 5000

print ("Test client sending packets to IP {0}, via                      port{1}\n".format(SERVER_IP, PORT_NUMBER))

sock = socket( AF_INET, SOCK_DGRAM )
sock.connect((SERVER_IP,PORT_NUMBER))

sock.send("Connecting...")
(msg,addr) = sock.recvfrom(15)
print(msg)

print "The categories are:"
categories = ['terrorism','jellikettu']

print (categories)
ans='dataset'
ans = ans.strip('\n')
if ans.endswith(' '):
    ans = ans.rstrip(' ')
twenty_train = load_files(ans, description=None, categories=categories, load_content=True, shuffle=True, encoding='utf-8', decode_error='ignore', random_state=42)  


count_vect = CountVectorizer()
X_train_counts = count_vect.fit_transform(twenty_train.data)
sock.sendto(pickle.dumps(twenty_train.target),addr)
(ms,addr) = sock.recvfrom(2000000)

tfidf_transformer = TfidfTransformer()
X_train_tfidf = tfidf_transformer.fit_transform(X_train_counts)
sock.sendto(pickle.dumps(X_train_tfidf),addr)

(ans,addr) = sock.recvfrom(2000)
docs_new=[ans]

X_new_counts = count_vect.transform(docs_new)
X_new_tfidf = tfidf_transformer.transform(X_new_counts)
sock.sendto(pickle.dumps(X_new_tfidf),addr)
(m,addr) = sock.recvfrom(2000000)

sock.sendto(pickle.dumps(twenty_train.target_names),addr)

print >>sys.stderr, 'closing socket'
sock.close()

sys.exit()

错误

Traceback (most recent call last):
  File "cl.py", line 43, in <module>
    sock.sendto(pickle.dumps(X_train_tfidf),addr)
socket.error: [Errno 90] Message too long

Answer 1

是。您应该使用SOCK_STREAM（TCP）套接字来发送大数据。使用SOCK_DGRAM（UDP）意味着每条消息都是独立的，并且必须符合UDP数据报的最大大小（仅低于64K）。但是，如果您使用TCP会话，则可以传输的大小没有限制。

但需要对单个消息进行构架，因为TCP不会维护消息边界。这通常通过在消息前面发送某种头部来完成，以便接收器知道在解码之前要读取多少。在您的情况下，您需要确保在调用pickle.loads之前收到整个数据块。标头可以像包含剩余消息长度的单个32位整数一样简单。（可能最好把它放在二进制文件中，以便你知道它 [长度]有多大。你可以使用struct模块pack来做到这一点。 unpack。）

另一种方法是简单地为每个要发送的数据块创建一个全新的连接：即连接，发送所有数据，关闭。这样，接收器可以直接接收到EOF，此时它知道它有整个数据块。

将大量数据从一个系统发送到另一个系统

1 个答案: