Question

我有一个包含大量文本文件的文件夹。每个都是gzip压缩，并称重几千兆字节。我编写了一段代码来分割每个gzip文件的内容：每个gzip文件都以gzip打开，然后读取每个指定的行块并将其写入新的gzip文件。

以下是文件file_compression.py中的代码：

import sys, os, file_manipulation as fm
import gzip


def splitGzipFile(fileName, dest=None, chunkPerSplit=100, linePerChunk=4, file_field_separator="_", zfill=3
                  , verbose=False, file_permission=None, execute=True):
    """
    Splits a gz file into chunk files.
    :param fileName:
    :param chunkPerSplit:
    :param linePerChunk:
    :return:
    """
    absPath = os.path.abspath(fileName)
    baseName = os.path.basename(absPath)
    dirName = os.path.dirname(absPath)
    destFolder = dirName if dest is None else dest


    ## Compute file fields
    rawBaseName, extensions = baseName.split(os.extsep, 1)

    if not str(extensions).startswith("."):
        extensions = "." + extensions

    file_fields = str(rawBaseName).split(file_field_separator)
    first_fields = file_fields[:-1] if file_fields.__len__() > 1 else file_fields
    first_file_part = file_field_separator.join(first_fields)
    last_file_field = file_fields[-1] if file_fields.__len__() > 1 else ""
    current_chunk = getCurrentChunkNumber(last_file_field)
    if current_chunk is None or current_chunk < 0:
        first_file_part = rawBaseName

    ## Initialize chunk variables
    linePerSplit = chunkPerSplit * linePerChunk
    # chunkCounter = 0

    chunkCounter = 0 if current_chunk is None else current_chunk-1

    for chunk in getFileChunks(fileName, linePerSplit):
        print "writing " + str(str(chunk).__len__()) + " ..."
        chunkCounter += 1
        oFile = fm.buildPath(destFolder) + first_file_part + file_field_separator + str(chunkCounter).zfill(zfill) + extensions

        if execute:
            writeGzipFile(oFile, chunk, file_permission)
        if verbose:
            print "Splitting: created file ", oFile



def getCurrentChunkNumber(chunk_field):
    """
    Tries to guess an integer from a string.
    :param chunk_field:
    :return: an integer, None if failure.
    """
    try:
        return int(chunk_field)
    except ValueError:
        return None


def getFileChunks(fileName, linePerSplit):
    with gzip.open(fileName, 'rb') as f:
        print "gzip open"
        lineCounter = 0
        currentChunk = ""
        for line in f:
            currentChunk += line
            lineCounter += 1
            if lineCounter >= linePerSplit:
                yield currentChunk
                currentChunk = ""
                lineCounter = 0
        if not currentChunk == '':
            yield currentChunk


def writeGzipFile(file_name, content, file_permission=None):
    import gzip
    with gzip.open(file_name, 'wb') as f:
        if not content == '':
            f.write(content)

    if file_permission is not None and type(file_permission) == int:
        os.chmod(file_name, file_permission)

此任务是多进程，在分割之前为每个文件创建一个进程。在删除之前，每个文件都是打开并拆分一次，我通过将它们记录在列表中来确保这一点：

from tools.file_utils import file_compression as fc, file_manipulation as fm
import multiprocessing
from multiprocessing import Process, Queue, Manager

manager = Manager()
split_seen = manager.list()

files = [...] # list is full of gzip files.
processList = []
sampleDir = "sample/dir/"

for file in files:
    fielPath = sampleDir + str(file)
    p = Process(target=processFile, args=(filePath, sampleDir, True))
    p.start()
    processList.append(p)

## Join the processes
for p in processList:
    p.join()

def processFile(filePath, destFolder, verbose=True):
    global split_seen
    if filePath in split_seen:
        print "Duplicate file processed: " + str(filePath)
        time.sleep(3)
    print "adding", filePath, split_seen.__len__()
    split_seen.append(filePath)
    fc.splitGzipFile(filePath, dest=destFolder, chunkPerSplit=4000000\
                                 , linePerChunk=4
                                 , verbose=True
                                 , file_permission=0770
                                 , zfill=3
                         )

    os.remove(filePath)

到目前为止，代码总是正常运行。但今天我遇到了gzip文件的问题＆＃39; CRC腐败：

Process Process-3:72:

Traceback (most recent call last):

  ...

  File "/.../tools/file_utils/file_compression.py", line 43, in splitGzipFile

    for chunk in getFileChunks(fileName, linePerSplit):

  File "/.../tools/file_utils/file_compression.py", line 70, in getFileChunks

    for line in f:

  File "/.../python2.7/lib/python2.7/gzip.py", line 450, in readline

    c = self.read(readsize)

  File "/.../python2.7/lib/python2.7/gzip.py", line 256, in read

    self._read(readsize)

  File "/.../python2.7/lib/python2.7/gzip.py", line 320, in _read

    self._read_eof()

  File "/.../python2.7/lib/python2.7/gzip.py", line 342, in _read_eof

    hex(self.crc)))

IOError: CRC check failed 0xddbb6045 != 0x34fd5580L

这个问题可能是什么原因？我必须再次说明它到目前为止总是工作，文件夹和文件总是相同的结构。这个例子的不同之处可能是我的脚本处理的gzip文件比平时多，可能是两倍。

是否可以同时访问相同的文件？但是我非常怀疑，我通过注册我的split_seen列表中访问的每个文件来确保不是这种情况。

我会采取任何暗示，因为我没有更多线索到哪里看。

编辑1

也许某些打开的文件被其他人或其他程序访问过？我不能要求并依赖于推荐书。首先，如果我要放multiprocess.Lock，是否会阻止任何其他线程，进程，程序，用户等修改文件？或者仅限于Python？我找不到任何关于此的文件。

Answer 1

我在运行了几个月的代码上得到了完全相同的错误。原来该文件源已损坏该特定文件。我回到了一个旧文件，它工作正常，我使用了一个较新的文件，它也工作正常。

Answer 2

我有同样的问题。我只是删除了旧文件，然后重新运行了代码。

rm -rf /tmp/imagenet/

HTH

Python gzip CRC检查失败

2 个答案: