如何在golang中读取utf16文本文件到字符串？

时间：2013-04-03 09:38:35

标签： unicode go readline utf-16

我可以将文件读取为bytes array

但当我将其转换为字符串

时

它将utf16字节视为ascii

如何正确转换？

package main

import ("fmt"
"os"
"bufio"
)

func main(){
    // read whole the file
    f, err := os.Open("test.txt")
    if err != nil {
        fmt.Printf("error opening file: %v\n",err)
        os.Exit(1)
    }
    r := bufio.NewReader(f)
    var s,b,e = r.ReadLine()
    if e==nil{
        fmt.Println(b)
        fmt.Println(s)
        fmt.Println(string(s))
    }
}

输出：

假

[255 254 91 0 83 0 99 0 114 0 105 0 112 0 116 0 32 0 73 0 110 0 102 0 111 0 93 0  13 0]

S c r i p t I n f o]

更新

在我测试了这两个例子后，我已经明确了现在的确切问题。

在Windows中，如果我在行尾添加换行符（CR + LF），则会在行中读取CR。因为readline函数无法正确处理unicode（[OD OA] = ok，[OD 00 OA 00] =不行）。

如果readline函数可以识别unicode，它应该理解[OD 00 OA 00]并返回[] uint16而不是[]字节。

所以我认为我不应该使用bufio.NewReader，因为它无法读取utf16，我没有看到bufio.NewReader.ReadLine可以接受参数作为标志来指示读取文本是utf8，utf16le / be或者utf32。 go库中是否有unicode文本的readline函数？

5 个答案:

答案 0 :(得分：12)

UTF16，UTF8和字节顺序标记由Unicode Consortium：UTF-16 FAQ，UTF-8 FAQ和Byte Order Mark (BOM) FAQ定义。

  

Issue 4802: bufio: reading lines is too cumbersome

     

在Go中读取文件中的行太麻烦了。

     

人们经常被吸引到bufio.Reader.ReadLine，因为它的名字，   但它有一个奇怪的签名，返回（line [] byte，isPrefix bool，   错误的错误），需要大量的工作。

     

ReadSlice和ReadString需要一个分隔符字节，这几乎是   永远是明显的和难看的'\ n'，也可以返回两行   和一个EOF

  

Revision: f685026a2d38

     

bufio：新的扫描仪界面

     

为扫描（可能是文本）数据添加一个新的简单界面，   基于一种名为Scanner的新类型。它有自己的内部   缓冲，所以即使不注射也应该合理有效   bufio.Reader。输入的格式由“拆分”定义   函数“，默认情况下分成行。

  

go1.1beta1 released

     

您可以从通常的位置下载二进制和源代码分发：       https://code.google.com/p/go/downloads/list?q=go1.1beta1

这是一个使用Unicode规则将UTF16文本文件行转换为Go UTF8编码字符串的程序。代码已经过修改，以利用Go 1.1中的新bufio.Scanner接口。

package main

import (
    "bufio"
    "bytes"
    "encoding/binary"
    "fmt"
    "os"
    "runtime"
    "unicode/utf16"
    "unicode/utf8"
)

// UTF16BytesToString converts UTF-16 encoded bytes, in big or little endian byte order,
// to a UTF-8 encoded string.
func UTF16BytesToString(b []byte, o binary.ByteOrder) string {
    utf := make([]uint16, (len(b)+(2-1))/2)
    for i := 0; i+(2-1) < len(b); i += 2 {
        utf[i/2] = o.Uint16(b[i:])
    }
    if len(b)/2 < len(utf) {
        utf[len(utf)-1] = utf8.RuneError
    }
    return string(utf16.Decode(utf))
}

// UTF-16 endian byte order
const (
    unknownEndian = iota
    bigEndian
    littleEndian
)

// dropCREndian drops a terminal \r from the endian data.
func dropCREndian(data []byte, t1, t2 byte) []byte {
    if len(data) > 1 {
        if data[len(data)-2] == t1 && data[len(data)-1] == t2 {
            return data[0 : len(data)-2]
        }
    }
    return data
}

// dropCRBE drops a terminal \r from the big endian data.
func dropCRBE(data []byte) []byte {
    return dropCREndian(data, '\x00', '\r')
}

// dropCRLE drops a terminal \r from the little endian data.
func dropCRLE(data []byte) []byte {
    return dropCREndian(data, '\r', '\x00')
}

// dropCR drops a terminal \r from the data.
func dropCR(data []byte) ([]byte, int) {
    var endian = unknownEndian
    switch ld := len(data); {
    case ld != len(dropCRLE(data)):
        endian = littleEndian
    case ld != len(dropCRBE(data)):
        endian = bigEndian
    }
    return data, endian
}

// SplitFunc is a split function for a Scanner that returns each line of
// text, stripped of any trailing end-of-line marker. The returned line may
// be empty. The end-of-line marker is one optional carriage return followed
// by one mandatory newline. In regular expression notation, it is `\r?\n`.
// The last non-empty line of input will be returned even if it has no
// newline.
func ScanUTF16LinesFunc(byteOrder binary.ByteOrder) (bufio.SplitFunc, func() binary.ByteOrder) {

    // Function closure variables
    var endian = unknownEndian
    switch byteOrder {
    case binary.BigEndian:
        endian = bigEndian
    case binary.LittleEndian:
        endian = littleEndian
    }
    const bom = 0xFEFF
    var checkBOM bool = endian == unknownEndian

    // Scanner split function
    splitFunc := func(data []byte, atEOF bool) (advance int, token []byte, err error) {

        if atEOF && len(data) == 0 {
            return 0, nil, nil
        }

        if checkBOM {
            checkBOM = false
            if len(data) > 1 {
                switch uint16(bom) {
                case uint16(data[0])<<8 | uint16(data[1]):
                    endian = bigEndian
                    return 2, nil, nil
                case uint16(data[1])<<8 | uint16(data[0]):
                    endian = littleEndian
                    return 2, nil, nil
                }
            }
        }

        // Scan for newline-terminated lines.
        i := 0
        for {
            j := bytes.IndexByte(data[i:], '\n')
            if j < 0 {
                break
            }
            i += j
            switch e := i % 2; e {
            case 1: // UTF-16BE
                if endian != littleEndian {
                    if i > 1 {
                        if data[i-1] == '\x00' {
                            endian = bigEndian
                            // We have a full newline-terminated line.
                            return i + 1, dropCRBE(data[0 : i-1]), nil
                        }
                    }
                }
            case 0: // UTF-16LE
                if endian != bigEndian {
                    if i+1 < len(data) {
                        i++
                        if data[i] == '\x00' {
                            endian = littleEndian
                            // We have a full newline-terminated line.
                            return i + 1, dropCRLE(data[0 : i-1]), nil
                        }
                    }
                }
            }
            i++
        }

        // If we're at EOF, we have a final, non-terminated line. Return it.
        if atEOF {
            // drop CR.
            advance = len(data)
            switch endian {
            case bigEndian:
                data = dropCRBE(data)
            case littleEndian:
                data = dropCRLE(data)
            default:
                data, endian = dropCR(data)
            }
            if endian == unknownEndian {
                if runtime.GOOS == "windows" {
                    endian = littleEndian
                } else {
                    endian = bigEndian
                }
            }
            return advance, data, nil
        }

        // Request more data.
        return 0, nil, nil
    }

    // Endian byte order function
    orderFunc := func() (byteOrder binary.ByteOrder) {
        switch endian {
        case bigEndian:
            byteOrder = binary.BigEndian
        case littleEndian:
            byteOrder = binary.LittleEndian
        }
        return byteOrder
    }

    return splitFunc, orderFunc
}

func main() {
    file, err := os.Open("utf16.le.txt")
    if err != nil {
        fmt.Println(err)
        os.Exit(1)
    }
    defer file.Close()
    fmt.Println(file.Name())

    rdr := bufio.NewReader(file)
    scanner := bufio.NewScanner(rdr)
    var bo binary.ByteOrder // unknown, infer from data
    // bo = binary.LittleEndian // windows
    splitFunc, orderFunc := ScanUTF16LinesFunc(bo)
    scanner.Split(splitFunc)

    for scanner.Scan() {
        b := scanner.Bytes()
        s := UTF16BytesToString(b, orderFunc())
        fmt.Println(len(s), s)
        fmt.Println(len(b), b)
    }
    fmt.Println(orderFunc())

    if err := scanner.Err(); err != nil {
        fmt.Println(err)
    }
}

输出：

utf16.le.txt
15 "Hello, 世界"
22 [34 0 72 0 101 0 108 0 108 0 111 0 44 0 32 0 22 78 76 117 34 0]
0 
0 []
15 "Hello, 世界"
22 [34 0 72 0 101 0 108 0 108 0 111 0 44 0 32 0 22 78 76 117 34 0]
LittleEndian

utf16.be.txt
15 "Hello, 世界"
22 [0 34 0 72 0 101 0 108 0 108 0 111 0 44 0 32 78 22 117 76 0 34]
0 
0 []
15 "Hello, 世界"
22 [0 34 0 72 0 101 0 108 0 108 0 111 0 44 0 32 78 22 117 76 0 34]
BigEndian

答案 1 :(得分：12)

golang.org/x/text/encoding/unicode的最新版本可以更轻松地执行此操作，因为它包含unicode.BOMOverride，可以智能地解释BOM。

这是ReadFileUTF16（），它与os.ReadFile（）类似，但解码为UTF-16。 

package main

import (
    "bytes"
    "fmt"
    "io/ioutil"
    "log"
    "strings"

    "golang.org/x/text/encoding/unicode"
    "golang.org/x/text/transform"
)

// Similar to ioutil.ReadFile() but decodes UTF-16.  Useful when
// reading data from MS-Windows systems that generate UTF-16BE files,
// but will do the right thing if other BOMs are found.
func ReadFileUTF16(filename string) ([]byte, error) {

    // Read the file into a []byte:
    raw, err := ioutil.ReadFile(filename)
    if err != nil {
        return nil, err
    }

    // Make an tranformer that converts MS-Win default to UTF8:
    win16be := unicode.UTF16(unicode.BigEndian, unicode.IgnoreBOM)
    // Make a transformer that is like win16be, but abides by BOM:
    utf16bom := unicode.BOMOverride(win16be.NewDecoder())

    // Make a Reader that uses utf16bom:
    unicodeReader := transform.NewReader(bytes.NewReader(raw), utf16bom)

    // decode and print:
    decoded, err := ioutil.ReadAll(unicodeReader)
    return decoded, err
}

func main() {
    data, err := ReadFileUTF16("inputfile.txt")
    if err != nil {
        log.Fatal(err)
    }
    final := strings.Replace(string(data), "\r\n", "\n", -1)
    fmt.Println(final)

}

这是NewScannerUTF16，它与os.Open（）类似，但返回扫描程序。

package main

import (
    "bufio"
    "fmt"
    "log"
    "os"

    "golang.org/x/text/encoding/unicode"
    "golang.org/x/text/transform"
)

type utfScanner interface {
    Read(p []byte) (n int, err error)
}

// Creates a scanner similar to os.Open() but decodes the file as UTF-16.
// Useful when reading data from MS-Windows systems that generate UTF-16BE
// files, but will do the right thing if other BOMs are found.
func NewScannerUTF16(filename string) (utfScanner, error) {

    // Read the file into a []byte:
    file, err := os.Open(filename)
    if err != nil {
        return nil, err
    }

    // Make an tranformer that converts MS-Win default to UTF8:
    win16be := unicode.UTF16(unicode.BigEndian, unicode.IgnoreBOM)
    // Make a transformer that is like win16be, but abides by BOM:
    utf16bom := unicode.BOMOverride(win16be.NewDecoder())

    // Make a Reader that uses utf16bom:
    unicodeReader := transform.NewReader(file, utf16bom)
    return unicodeReader, nil
}

func main() {

    s, err := NewScannerUTF16("inputfile.txt")
    if err != nil {
        log.Fatal(err)
    }

    scanner := bufio.NewScanner(s)
    for scanner.Scan() {
        fmt.Println(scanner.Text()) // Println will add back the final '\n'
    }
    if err := scanner.Err(); err != nil {
        fmt.Fprintln(os.Stderr, "reading inputfile:", err)
    }

}

仅供参考：我已将这些功能纳入开源模块并进行了进一步改进。见https://github.com/TomOnTime/utfutil/

答案 2 :(得分：4)

例如：

package main

import (
        "errors"
        "fmt"
        "log"
        "unicode/utf16"
)

func utf16toString(b []uint8) (string, error) {
        if len(b)&1 != 0 {
                return "", errors.New("len(b) must be even")
        }

        // Check BOM
        var bom int
        if len(b) >= 2 {
                switch n := int(b[0])<<8 | int(b[1]); n {
                case 0xfffe:
                        bom = 1
                        fallthrough
                case 0xfeff:
                        b = b[2:]
                }
        }

        w := make([]uint16, len(b)/2)
        for i := range w {
                w[i] = uint16(b[2*i+bom&1])<<8 | uint16(b[2*i+(bom+1)&1])
        }
        return string(utf16.Decode(w)), nil
}

func main() {
        // Simulated data from e.g. a file
        b := []byte{255, 254, 91, 0, 83, 0, 99, 0, 114, 0, 105, 0, 112, 0, 116, 0, 32, 0, 73, 0, 110, 0, 102, 0, 111, 0, 93, 0, 13, 0}
        s, err := utf16toString(b)
        if err != nil {
                log.Fatal(err)
        }

        fmt.Printf("%q", s)
}

（还here）

输出：

"[Script Info]\r"

答案 3 :(得分：1)

这是最简单的阅读方法：

package main

import (
    "bufio"
    "fmt"
    "log"
    "os"

    "golang.org/x/text/encoding/unicode"
    "golang.org/x/text/transform"
)

func main() {
    file, err := os.Open("./text.txt")
    if err != nil {
        log.Fatal(err)
    }

    scanner := bufio.NewScanner(transform.NewReader(file, unicode.UTF16(unicode.LittleEndian, unicode.UseBOM).NewDecoder()))
    for scanner.Scan() {
        fmt.Printf(scanner.Text())
    }
}

由于Windows默认情况下使用link，因此我们使用unicode.UseBOM策略从文本中检索BOM，并使用unicode.LittleEndian作为后备广告

答案 4 :(得分：0)

如果您希望将任何内容打印为字符串，可以使用fmt.Sprint 

package main

import (
    "bufio"
    "fmt"
    "os"
)

func main() {
    // read whole the file
    f, err := os.Open("test.txt")
    if err != nil {
        fmt.Printf("error opening file: %v\n", err)
        os.Exit(1)
    }
    r := bufio.NewReader(f)
    var s, b, e = r.ReadLine()
    if e == nil {
        fmt.Println(b)
        fmt.Println(s)
        fmt.Println(fmt.Sprint(string(s)))
    }
}
相关问题
最新问题