Question

Python和wc对给定字符串的字节数（长度）有很大的不同意见：

with open("commedia.pfc", "w") as f:
    t = ''.join(chr(int(b, base=2)) for b in chunks(compressed, 8))
    print(len(t))
    f.write(t)

Output : 318885

$> wc commedia.pfc
  2181  12282 461491 commedia.pfc

该文件主要由不可读的字符组成，因此我将提供一个hexdump：

http://www.filedropper.com/dump_2

该文件是无前缀压缩的结果，如果您要求我可以提供生成它的完整代码以及输入文本。

为什么两个字节数都不相等？

我添加了压缩算法的完整代码，它看起来很长但是充满了文档和测试，所以应该很容易理解：

"""
Implementation of prefix-free compression and decompression.
"""
import doctest
from itertools import islice
from collections import Counter
import random
import json

def binary_strings(s):
    """
    Given an initial list of binary strings `s`,
    yield all binary strings ending in one of `s` strings.

    >>> take(9, binary_strings(["010", "111"]))
    ['010', '111', '0010', '1010', '0111', '1111', '00010', '10010', '01010']
    """
    yield from s
    while True:
        s = [b + x for x in s for b in "01"]
        yield from s

def take(n, iterable):
    """
    Return first n items of the iterable as a list.
    """
    return list(islice(iterable, n))

def chunks(xs, n, pad='0'):
    """
    Yield successive n-sized chunks from xs.
    """
    for i in range(0, len(xs), n):
        yield xs[i:i + n]

def reverse_dict(dictionary):
    """
    >>> sorted(reverse_dict({1:"a",2:"b"}).items())
    [('a', 1), ('b', 2)]
    """
    return {value : key for key, value in dictionary.items()}

def prefix_free(generator):
    """
    Given a `generator`, yield all the items from it
    that do not start with any preceding element.

    >>> take(6, prefix_free(binary_strings(["00", "01"])))
    ['00', '01', '100', '101', '1100', '1101']
    """
    seen = []
    for x in generator:
        if not any(x.startswith(i) for i in seen):
            yield x
            seen.append(x)

def build_translation_dict(text, starting_binary_codes=["000", "100","111"]):
    """
    Builds a dict for `prefix_free_compression` where
       More common char -> More short binary strings
    This is compression as the shorter binary strings will be seen more times than
    the long ones.

    Univocity in decoding is given by the binary_strings being prefix free.

    >>> sorted(build_translation_dict("aaaaa bbbb ccc dd e", ["01", "11"]).items())
    [(' ', '001'), ('a', '01'), ('b', '11'), ('c', '101'), ('d', '0001'), ('e', '1001')]
    """
    binaries = sorted(list(take(len(set(text)), prefix_free(binary_strings(starting_binary_codes)))), key=len)
    frequencies = Counter(text)
    # char value tiebreaker to avoid non-determinism                     v
    alphabet = sorted(list(set(text)), key=(lambda ch: (frequencies[ch], ch)), reverse=True)
    return dict(zip(alphabet, binaries))

def prefix_free_compression(text, starting_binary_codes=["000", "100","111"]):
    """
    Implements `prefix_free_compression`, simply uses the dict
    made with `build_translation_dict`.

    Returns a tuple (compressed_message, tranlation_dict) as the dict is needed
    for decompression.

    >>> prefix_free_compression("aaaaa bbbb ccc dd e", ["01", "11"])[0]
    '010101010100111111111001101101101001000100010011001'
    """
    translate = build_translation_dict(text, starting_binary_codes)
    # print(translate)
    return ''.join(translate[i] for i in text), translate

def prefix_free_decompression(compressed, translation_dict):
    """
    Decompresses a prefix free `compressed` message in the form of a string
    composed only of '0' and '1'.

    Being the binary codes prefix free,
    the decompression is allowed to take the earliest match it finds.

    >>> message, d = prefix_free_compression("aaaaa bbbb ccc dd e", ["01", "11"])
    >>> message
    '010101010100111111111001101101101001000100010011001'
    >>> sorted(d.items())
    [(' ', '001'), ('a', '01'), ('b', '11'), ('c', '101'), ('d', '0001'), ('e', '1001')]
    >>> ''.join(prefix_free_decompression(message, d))
    'aaaaa bbbb ccc dd e'
    """
    decoding_translate = reverse_dict(translation_dict)
    # print(decoding_translate)
    word = ''
    for bit in compressed:
        # print(word, "-", bit)
        if word in decoding_translate:
            yield decoding_translate[word]
            word = ''
        word += bit
    yield decoding_translate[word]


if __name__ == "__main__":
    doctest.testmod()
    with open("commedia.txt") as f:
        text = f.read()
    compressed, d = prefix_free_compression(text)
    with open("commedia.pfc", "w") as f:
        t = ''.join(chr(int(b, base=2)) for b in chunks(compressed, 8))
        print(len(t))
        f.write(t)
    with open("commedia.pfcd", "w") as f:
        f.write(json.dumps(d))
    # dividing by 8 goes from bit length to byte length
    print("Compressed / uncompressed ratio is {}".format((len(compressed)//8) / len(text)))
    original = ''.join(prefix_free_decompression(compressed, d))
    assert original == text

commedia.txt是filedropper.com/commedia

Answer 1

您正在使用Python3和str对象 - 这意味着您在len(t)中看到的计数是字符串中字符的数量。现在，字符不是字节 - and it is so since the 90's。

由于您没有声明显式文本编码，因此文件编写使用系统默认编码对您的文本进行编码 - 在Linux或Mac OS X上将是utf-8 - 一种编码，其中任何字符都不属于ASCII范围（ord（ch）> 127）在磁盘上使用多个字节。

所以，你的程序基本上是错误的。首先，定义您是否正在处理 text 或 bytes 。如果您要处理字节，请打开文件以便以二进制模式（wb，而不是w）进行写入并更改此行：

t = ''.join(chr(int(b, base=2)) for b in chunks(compressed, 8))

到

t = bytes((int(b, base=2) for b in chunks(compressed, 8))

这样很明显，你正在处理字节本身，而不是破坏字符和字节。

当然，有一个丑陋的解决方法来做一个＆＃34;透明编码＆＃34;你有一个字节对象的文本 - （如果你的原始列表将包含0-256范围内的所有字符代码点，那就是）：你可以在编写之前用t编码对你之前的latin1进行编码它到一个文件。但这在语义上是错误的。

你也可以尝试Python的鲜为人知的＆＃34; bytearray＆＃34; object：它使一个人能够处理8位数字的元素，并且具有可变和可扩展的便利性（就像C＆＃34;字符串＆＃34;预先分配足够的内存空间）

Answer 2

@jsbueno是对的。此外，如果您在二进制读取模式下打开生成的文件，您将获得良好的结果：

#include<stdio.h>
#include<limits.h>

int array[5] = {1, 3, 2, 5, 4};

int min = INT_MAX;
int max = INT_MIN;

int main(void)
{
  for (size_t i = 0; i < sizeof(array) / sizeof(*array); i++) {
    int x = array[i];
    if (x < min) min = x;
    if (x > max) max = x;
  }
  printf("The minimum is %i", min);
  printf("The maximum is %i", max);
}

为什么Python和wc对字节数不一致？

2 个答案: