encutil.py

Question

继上一个pycryptodome问题之后，我的要求现在已更改为支持 90G 数据加密。因此，我做了一些设计更改，对加密代码进行了分解，并使它们全部在子进程中运行。

tar zcvf - /array22/vol4/home | openssl des3 -salt | dd of=/dev/st0

上述想法是由here

触发的

现在我有2个文件：

encutil.py

#!/usr/bin/python

import sys, os, pwd
from Crypto.PublicKey import RSA
from Crypto.Cipher import AES, PKCS1_OAEP
from Crypto.Random import get_random_bytes

symmetric_key = get_random_bytes(16 * 2)
cipher_rsa = PKCS1_OAEP.new(RSA.import_key(open("./public.pem").read()))
enc_symmetric_key = cipher_rsa.encrypt(symmetric_key)
cipher = AES.new(symmetric_key, AES.MODE_GCM)
[sys.stdout.write(x) for x in (enc_symmetric_key, cipher.nonce,"".join(reversed(cipher.encrypt_and_digest(sys.stdin.read()))))]

main.py

#! /usr/bin/python

import os, sys, time
import tarfile, StringIO, time
from subprocess import Popen, PIPE, call
from Crypto.PublicKey import RSA
from Crypto.Cipher import AES, PKCS1_OAEP
from Crypto.Random import get_random_bytes

print "Start time %s"%time.time()
try:
    p1=Popen("tar -czf - ./src", shell=True, stdout=PIPE)
    p2=Popen("python ./encutil.py", shell=True, stdin=p1.stdout, stdout=PIPE)
    FNULL = open(os.devnull, 'w')
    p3=Popen("/bin/dd bs=10M iflag=fullblock oflag=direct,sync conv=fsync,notrunc,noerror status=progress of=./data.bin", shell=True, stdin=p2.stdout, stderr=FNULL)
    p3.wait()
except Exception,e:
    raise str(e)
finally:
    p2.stdout.close()
    p1.stdout.close()

def doRestore():
        try:
            privKey = RSA.import_key(open("./private.pem").read())
            cipher_rsa = PKCS1_OAEP.new(privKey)
            file_in = open("./data.bin", "rb")
            enc_symmetric_key, nonce, tag, ciphertext = [file_in.read(x) for x in (privKey.size_in_bytes(), 16, 16, -1)]
            symmetric_key = cipher_rsa.decrypt(enc_symmetric_key)
            cipher = AES.new(symmetric_key, AES.MODE_GCM, nonce)
            tar = tarfile.open(fileobj=StringIO.StringIO(cipher.decrypt_and_verify(ciphertext, tag)), mode='r|*')
            tar.extractall(path='./dst')
        except Exception,e:
            print e
        finally:
            if file_in != None:
                file_in.close()
            if tar != None:
                tar.close()
            os.remove("./data.bin")

doRestore()
print "End time %s"%time.time()

假定公钥和私钥均可用并且就位。

而且，当我在执行了一段时间后执行以下命令时，出现错误： 长度不正确的密文 ，而没有任何回溯：

/usr/bin/systemd-run --scope -p MemoryLimit=80G ./main.py

但是它可以成功运行较少的数据输入，例如 40G 数据

我的系统详细信息是：

HW: HP ProLiant DL360 Gen10 with more than 500G of HDD space and 125G of RAM
OS: RHEL7.4 64-bit Kernel: 3.10.0-693.el7.x86_64
Python version: 2.7.5
Pycryptodome version: 3.7.2

如果我不通过 systemd-run 控制内存资源，那么Python会在执行的某个时刻抛出 MemoryError ，并以与“ < em>长度不正确的密文。 ”消息

Traceback (most recent call last):
  File "./encutil.py", line 12, in <module>
    [sys.stdout.write(x) for x in (enc_symmetric_key, cipher.nonce,"".join(reversed(cipher.encrypt_and_digest(sys.stdin.read()))))]
  File "/opt/LEBackupandRestore/lib/3pp/Crypto/Cipher/_mode_gcm.py", line 547, in encrypt_and_digest
    return self.encrypt(plaintext, output=output), self.digest()
  File "/opt/LEBackupandRestore/lib/3pp/Crypto/Cipher/_mode_gcm.py", line 374, in encrypt
    ciphertext = self._cipher.encrypt(plaintext, output=output)
  File "/opt/LEBackupandRestore/lib/3pp/Crypto/Cipher/_mode_ctr.py", line 211, in encrypt
    return get_raw_buffer(ciphertext)
  File "/opt/LEBackupandRestore/lib/3pp/Crypto/Util/_raw_api.py", line 187, in get_raw_buffer
    return buf.raw
MemoryError
Ciphertext with incorrect length.

我无法从stackoverflow

中已经提出的解决方案中获得任何线索

更改之前的原始代码设计如下：

#! /usr/bin/python    
import os, pwd, sys
from subprocess import Popen, PIPE, check_call
from BackupRestoreException import BackupRestoreException, ErrorCode
from Crypto.PublicKey import RSA
from Crypto.Cipher import AES, PKCS1_OAEP
from Crypto.Random import get_random_bytes
from Crypto.Util.Padding import pad,unpad
import tarfile,StringIO,time

# Key Generation
key = RSA.generate(2048)
private_key = key.export_key()
file_out = open("private.pem", "wb")
file_out.write(private_key)
file_out.close()

public_key = key.publickey().export_key()
file_out = open("public.pem", "wb")
file_out.write(public_key)
file_out.close()

public_key = RSA.import_key(open("public.pem").read())
session_key = get_random_bytes(16)
cipher_rsa = PKCS1_OAEP.new(public_key)
enc_session_key = cipher_rsa.encrypt(session_key)

def archiveData():
    cmd = ["tar", "--acls", "--selinux", "-zcPf", "-", "./src"]
    return Popen(cmd,stdout=PIPE).communicate()[0]

# Encryption
cipher_aes = AES.new(session_key, AES.MODE_EAX)
ciphertext, tag = cipher_aes.encrypt_and_digest(archiveData())
file_out = open("data.bin", "wb")
[ file_out.write(x) for x in (enc_session_key, cipher_aes.nonce, tag, ciphertext) ]
file_out.close()


# Decryption
private_key = RSA.import_key(open("private.pem").read())
file_in = open("data.bin", "rb")
enc_session_key, nonce, tag, ciphertext = [ file_in.read(x) for x in (private_key.size_in_bytes(), 16, 16, -1) ]
file_in.close()
cipher_rsa = PKCS1_OAEP.new(private_key)
session_key = cipher_rsa.decrypt(enc_session_key)
cipher = AES.new(session_key, AES.MODE_EAX, nonce)
tar = tarfile.open(fileobj=StringIO.StringIO(cipher.decrypt_and_verify(ciphertext, tag)), mode='r|*')
os.chdir("/home/cfuser/target")
tar.extractall(path='.')

Answer 1

我已经解决了这个问题，下面的代码适用于更大的数据大小。仍然欢迎任何代码注释/改进想法/建议！

#! /usr/bin/python

import os, time, tarfile, io
from subprocess import Popen, PIPE, check_call
from Crypto.PublicKey import RSA
from Crypto.Cipher import AES, PKCS1_OAEP
from Crypto.Random import get_random_bytes

print "****** Start time %s" % time.time()

BLOCK_SIZE = 16
BIN_FILE = "/nfs/data.bin"
symmetric_key = get_random_bytes(BLOCK_SIZE * 2)
enc_symmetric_key = PKCS1_OAEP.new(RSA.import_key(open("./public.pem").read())).encrypt(symmetric_key)
cipher_rsa_prikey = PKCS1_OAEP.new(RSA.import_key(open("./private.pem").read()))

chunk_size = BLOCK_SIZE * 1024 * 1024 + BLOCK_SIZE
tag_size = BLOCK_SIZE
ciphertxt_size = chunk_size - tag_size
nonce_size = BLOCK_SIZE
enc_key_size = RSA.import_key(open("./private.pem").read()).size_in_bytes() # 256

def runTarCommand():
    cmd = "/usr/bin/systemd-run -q --scope -p MemoryLimit=10G tar -czPf - /root/src"
    return Popen(cmd, bufsize=chunk_size, shell=True, stdout=PIPE)

def doNFSBackup():
    try:
        p = runTarCommand()
        with open(BIN_FILE,'wb') as f:
            f.write(enc_symmetric_key)
            while True:
               dataChunk = p.stdout.read(ciphertxt_size)
               if dataChunk:
                  cipher = AES.new(symmetric_key, AES.MODE_GCM)
                  f.write(cipher.nonce + b"".join(reversed(cipher.encrypt_and_digest(dataChunk))))
               else:
                  break
    except Exception,e:
        print e
    finally:
        p.stdout.close()

def doNFSRestore():
    try:
        extractProc = Popen('tar -C /root/src -xzPf -', bufsize=8192, stdin=PIPE,shell=True)
        file_in = open(BIN_FILE, "rb")
        symmetric_key = cipher_rsa_prikey.decrypt(file_in.read(enc_key_size))
        nonce = file_in.read(nonce_size)
        while nonce:
            ciphertxtTag = file_in.read(chunk_size)
            cipher = AES.new(symmetric_key, AES.MODE_GCM, nonce)
            extractProc.stdin.write(cipher.decrypt_and_verify(ciphertxtTag[BLOCK_SIZE:], ciphertxtTag[:BLOCK_SIZE]))
            nonce = file_in.read(nonce_size)
    except Exception,e:
        print e
    finally:
        if file_in != None:
            file_in.close()
        if os.path.exists(BIN_FILE): os.remove(BIN_FILE)

def doTapeBackup():
    def tarinfoFun(tar, bytsIO):
        info = tarfile.TarInfo(name='test.tar')
        info.size = len(bytsIO.getvalue())
        info.mtime = time.time()
        info.mode = 0755
        tar.addfile(tarinfo=info, fileobj=bytsIO)

    try:
        cmd = "mt -f /dev/nst0 load; mt -f /dev/nst0 rewind; mt -f /dev/nst0 setblk %d" %(chunk_size + nonce_size)
        check_call(cmd, shell=True)
        p = runTarCommand()
        tar = tarfile.TarFile("/dev/nst0", "w")

        bytsIO = io.BytesIO()
        bytsIO.write(enc_symmetric_key)
        bytsIO.seek(0)
        tarinfoFun(tar, bytsIO)
        bytsIO.close()

        bytsIO = io.BytesIO()
        bytesread = 0
        while True:
            dataChunk = p.stdout.read(ciphertxt_size)
            if not dataChunk:
                if bytesread != 0:
                    bytsIO.seek(0)
                    tarinfoFun(tar, bytsIO)
                    bytsIO.close()
                p.communicate()
                break
            cipher = AES.new(symmetric_key, AES.MODE_GCM)
            bytsIO.write(cipher.nonce + b"".join(reversed(cipher.encrypt_and_digest(dataChunk))))
            bytesread += chunk_size + nonce_size
            if bytesread == (chunk_size + nonce_size) * 8:
                bytsIO.seek(0)
                tarinfoFun(tar, bytsIO)
                bytsIO.close()
                bytsIO = io.BytesIO()
                bytesread = 0
    except Exception,e:
        print e
    finally:
        tar.close()
        p.stdout.close()

def doTapeRestore():
    try:
        check_call("mt -f /dev/nst0 load; mt -f /dev/nst0 rewind", shell=True, stdout=PIPE)
        p1 = Popen("tar -xPf /dev/nst0 -O", shell=True, stdout=PIPE)
        p2 = Popen("tar -C /nfs -xzPf -", shell=True, stdin=PIPE)
        symmetric_key = cipher_rsa_prikey.decrypt(p1.stdout.read(enc_key_size))
        while True:
            ciphertxtTag = p1.stdout.read(chunk_size + nonce_size)
            if not ciphertxtTag:
                p2.communicate()
                break
            nonce = ciphertxtTag[:nonce_size]
            cipher = AES.new(symmetric_key, AES.MODE_GCM, nonce)
            p2.stdin.write(cipher.decrypt_and_verify(ciphertxtTag[32:], ciphertxtTag[BLOCK_SIZE:32]))
    except Exception,e:
        print e
    finally:
       pass

doNFSBackup()
doNFSRestore()
doTapeBackup()
doTapeRestore()

print "****** End time %s" % time.time()

Answer 2

查看我实现的rsa助手类 pycryptodome。

https://gist.github.com/kadaliao/4b111855037c8d2aad33627ae1f5817e

它可以处理大型消息的加密和解密。

不会看到这些错误：

“ ValueError：纯文本太长。”
“ ValueError：密文长度不正确。“

Python Pycryptodome加密抛出“长度不正确的密文”错误

encutil.py

main.py

2 个答案: