我有一组markdown文件要传递给jekyll项目，需要找到它们的编码格式，即带有BOM的UTF-8或没有BOM的UTF-8或使用程序或API的ANSI。
如果我传递文件的位置，则必须列出，读取文件，并且应该生成编码。
是否有任何代码或API？
我已经尝试过用于流阅读器的sr.CurrentEncoding，如有效查找任何文件的编码方式中所述但结果因记事本++结果而异。
还试图在https://github.com/errepi/ude中通过在c＃项目中实现ude.dll来使用https://social.msdn.microsoft.com/Forums/vstudio/en-US/862e3342-cc88-478f-bca2-e2de6f60d2fb/detect-encoding-of-the-file?forum=csharpgeneral（Mozilla Universal Charset Detector），但结果在notepad ++中无效，文件编码是显示为utf-8，但是从程序中，结果是带有BOM的utf-8。
但我应该从两个方面得到相同的结果，所以问题出现在哪里？
答案 0 :(得分：6)
检测编码始终是一件棘手的事情，但检测BOM很简单。要将BOM作为字节数组，只需使用编码对象的GetPreamble()函数。这应该允许您通过序言检测整个编码范围。
现在，至于在没有前导码的情况下检测UTF-8，实际上也不是很难。请参阅UTF8 has strict bitwise rules about what values are expected in a valid sequence，您可以初始化UTF8Encoding对象in a way that will fail by throwing an exception when these sequences are incorrect。
因此，如果您首先进行BOM检查，然后进行严格的解码检查，最后再回到Win-1252编码（您所谓的＆＃34; ANSI＆＃34;），那么您的检测就完成了。
Byte[] bytes = File.ReadAllBytes(filename);
Encoding encoding = null;
String text = null;
// Test UTF8 with BOM. This check can easily be copied and adapted
// to detect many other encodings that use BOMs.
UTF8Encoding encUtf8Bom = new UTF8Encoding(true, true);
Boolean couldBeUtf8 = true;
Byte[] preamble = encUtf8Bom.GetPreamble();
Int32 prLen = preamble.Length;
if (bytes.Length >= prLen && preamble.SequenceEqual(bytes.Take(prLen)))
{
// UTF8 BOM found; use encUtf8Bom to decode.
try
{
// Seems that despite being an encoding with preamble,
// it doesn't actually skip said preamble when decoding...
text = encUtf8Bom.GetString(bytes, prLen, bytes.Length - prLen);
encoding = encUtf8Bom;
}
catch (ArgumentException)
{
// Confirmed as not UTF-8!
couldBeUtf8 = false;
}
}
// use boolean to skip this if it's already confirmed as incorrect UTF-8 decoding.
if (couldBeUtf8 && encoding == null)
{
// test UTF-8 on strict encoding rules. Note that on pure ASCII this will
// succeed as well, since valid ASCII is automatically valid UTF-8.
UTF8Encoding encUtf8NoBom = new UTF8Encoding(false, true);
try
{
text = encUtf8NoBom.GetString(bytes);
encoding = encUtf8NoBom;
}
catch (ArgumentException)
{
// Confirmed as not UTF-8!
}
}
// fall back to default ANSI encoding.
if (encoding == null)
{
encoding = Encoding.GetEncoding(1252);
text = encoding.GetString(bytes);
}
请注意，Windows-1252（美国/西欧ANSI）是一个每字节一个字节的编码，这意味着它中的所有内容都会生成一个技术上有效的字符，因此unless you go for heuristic methods无需进一步检测它将它与其他每字符一个字节的编码区分开来。
答案 1 :(得分：2)
不断创新。
示例（DetectOrGuessEncoding）：
namespace SQLMerge
{
class EncodingDetector
{
public static System.Text.Encoding BomInfo(string srcFile)
{
return BomInfo(srcFile, false);
} // End Function BomInfo
public static System.Text.Encoding BomInfo(string srcFile, bool thorough)
{
byte[] b = new byte[5];
using (System.IO.FileStream file = new System.IO.FileStream(srcFile, System.IO.FileMode.Open, System.IO.FileAccess.Read, System.IO.FileShare.Read))
{
int numRead = file.Read(b, 0, 5);
if (numRead < 5)
System.Array.Resize(ref b, numRead);
file.Close();
} // End Using file
if (b.Length >= 4 && b[0] == 0x00 && b[1] == 0x00 && b[2] == 0xFE && b[3] == 0xFF)
return System.Text.Encoding.GetEncoding("utf-32BE"); // UTF-32, big-endian
else if (b.Length >= 4 && b[0] == 0xFF && b[1] == 0xFE && b[2] == 0x00 && b[3] == 0x00)
return System.Text.Encoding.UTF32; // UTF-32, little-endian
else if (b.Length >= 2 && b[0] == 0xFE && b[1] == 0xFF)
return System.Text.Encoding.BigEndianUnicode; // UTF-16, big-endian
else if (b.Length >= 2 && b[0] == 0xFF && b[1] == 0xFE)
return System.Text.Encoding.Unicode; // UTF-16, little-endian
else if (b.Length >= 3 && b[0] == 0xEF && b[1] == 0xBB && b[2] == 0xBF)
return System.Text.Encoding.UTF8; // UTF-8
else if (b.Length >= 3 && b[0] == 0x2b && b[1] == 0x2f && b[2] == 0x76)
return System.Text.Encoding.UTF7; // UTF-7
// Maybe there is a future encoding ...
// PS: The above yields more than this - this doesn't find UTF7 ...
if (thorough)
{
foreach (System.Text.EncodingInfo ei in System.Text.Encoding.GetEncodings())
{
System.Text.Encoding enc = ei.GetEncoding();
byte[] preamble = enc.GetPreamble();
if (preamble.Length == 0)
continue;
if (preamble.Length > b.Length)
continue;
for (int i = 0; i < preamble.Length; ++i)
{
if (b[i] != preamble[i])
{
goto NextEncoding;
}
} // Next i
return enc;
NextEncoding:
continue;
} // Next ei
} // End if (thorough)
return null;
} // End Function BomInfo
public static System.Text.Encoding DetectOrGuessEncoding(string fileName)
{
return DetectOrGuessEncoding(fileName, false);
}
public static System.Text.Encoding DetectOrGuessEncoding(string fileName, bool withOutput)
{
if (!System.IO.File.Exists(fileName))
return null;
System.ConsoleColor origBack = System.ConsoleColor.Black;
System.ConsoleColor origFore = System.ConsoleColor.White;
if (withOutput)
{
origBack = System.Console.BackgroundColor;
origFore = System.Console.ForegroundColor;
}
// System.Text.Encoding systemEncoding = System.Text.Encoding.Default; // Returns hard-coded UTF8 on .NET Core ...
System.Text.Encoding systemEncoding = GetSystemEncoding();
System.Text.Encoding enc = BomInfo(fileName);
if (enc != null)
{
if (withOutput)
{
System.Console.BackgroundColor = System.ConsoleColor.Green;
System.Console.ForegroundColor = System.ConsoleColor.White;
System.Console.WriteLine(fileName);
System.Console.WriteLine(enc);
System.Console.BackgroundColor = origBack;
System.Console.ForegroundColor = origFore;
}
return enc;
}
using (System.IO.Stream strm = System.IO.File.OpenRead(fileName))
{
UtfUnknown.DetectionResult detect = UtfUnknown.CharsetDetector.DetectFromStream(strm);
if (detect != null && detect.Details != null && detect.Details.Count > 0 && detect.Details[0].Confidence < 1)
{
if (withOutput)
{
System.Console.BackgroundColor = System.ConsoleColor.Red;
System.Console.ForegroundColor = System.ConsoleColor.White;
System.Console.WriteLine(fileName);
System.Console.WriteLine(detect);
System.Console.BackgroundColor = origBack;
System.Console.ForegroundColor = origFore;
}
foreach (UtfUnknown.DetectionDetail detail in detect.Details)
{
if (detail.Encoding == systemEncoding
|| detail.Encoding == System.Text.Encoding.UTF8
)
return detail.Encoding;
}
return detect.Details[0].Encoding;
}
else if (detect != null && detect.Details != null && detect.Details.Count > 0)
{
if (withOutput)
{
System.Console.BackgroundColor = System.ConsoleColor.Green;
System.Console.ForegroundColor = System.ConsoleColor.White;
System.Console.WriteLine(fileName);
System.Console.WriteLine(detect);
System.Console.BackgroundColor = origBack;
System.Console.ForegroundColor = origFore;
}
return detect.Details[0].Encoding;
}
enc = GetSystemEncoding();
if (withOutput)
{
System.Console.BackgroundColor = System.ConsoleColor.DarkRed;
System.Console.ForegroundColor = System.ConsoleColor.Yellow;
System.Console.WriteLine(fileName);
System.Console.Write("Assuming ");
System.Console.Write(enc.WebName);
System.Console.WriteLine("...");
System.Console.BackgroundColor = origBack;
System.Console.ForegroundColor = origFore;
}
return systemEncoding;
} // End Using strm
} // End Function DetectOrGuessEncoding
public static System.Text.Encoding GetSystemEncoding()
{
// The OEM code page for use by legacy console applications
// int oem = System.Globalization.CultureInfo.CurrentCulture.TextInfo.OEMCodePage;
// The ANSI code page for use by legacy GUI applications
// int ansi = System.Globalization.CultureInfo.InstalledUICulture.TextInfo.ANSICodePage; // Machine
int ansi = System.Globalization.CultureInfo.CurrentCulture.TextInfo.ANSICodePage; // User
try
{
// https://stackoverflow.com/questions/38476796/how-to-set-net-core-in-if-statement-for-compilation
#if ( NETSTANDARD && !NETSTANDARD1_0 ) || NETCORE || NETCOREAPP3_0 || NETCOREAPP3_1
System.Text.Encoding.RegisterProvider(System.Text.CodePagesEncodingProvider.Instance);
#endif
System.Text.Encoding enc = System.Text.Encoding.GetEncoding(ansi);
return enc;
}
catch (System.Exception)
{ }
try
{
foreach (System.Text.EncodingInfo ei in System.Text.Encoding.GetEncodings())
{
System.Text.Encoding e = ei.GetEncoding();
// 20'127: US-ASCII
if (e.WindowsCodePage == ansi && e.CodePage != 20127)
{
return e;
}
}
}
catch (System.Exception)
{ }
// return System.Text.Encoding.GetEncoding("iso-8859-1");
return System.Text.Encoding.UTF8;
} // End Function GetSystemEncoding
} // End Class
}
答案 2 :(得分：0)
就这样
var tempText = "Finanssivalvonnalla on syytä epäillä";
byte[] bytes = Encoding.GetEncoding("ISO-8859-9").GetBytes(tempText);
tempText = Encoding.UTF8.GetString(bytes);
tempText = WebUtility.HtmlDecode(tempText);
答案 3 :(得分：-1)
namespace WindowsFormsApp2
{
public partial class Form1 : Form
{
public Form1()
{
InitializeComponent();
}
private void button1_Click(object sender, EventArgs e)
{
List<FilePath> filePaths = new List<FilePath>();
filePaths = GetLstPaths();
}
public static List<FilePath> GetLstPaths()
{
#region Getting Files
DirectoryInfo directoryInfo = new DirectoryInfo(@"C:\Users\Safi\Desktop\ss\");
DirectoryInfo directoryTargetInfo = new DirectoryInfo(@"C:\Users\Safi\Desktop\ss1\");
FileInfo[] fileInfos = directoryInfo.GetFiles("*.txt");
List<FilePath> lstFiles = new List<FilePath>();
foreach (FileInfo fileInfo in fileInfos)
{
Encoding enco = GetLittleIndianFiles(directoryInfo + fileInfo.Name);
string filePath = directoryInfo + fileInfo.Name;
string targetFilePath = directoryTargetInfo + fileInfo.Name;
if (enco != null)
{
FilePath f1 = new FilePath();
f1.filePath = filePath;
f1.targetFilePath = targetFilePath;
lstFiles.Add(f1);
}
}
int count = 0;
lstFiles.ForEach(d =>
{
count++;
});
MessageBox.Show(Convert.ToString(count) + "Files are Converted");
#endregion
return lstFiles;
}
public static Encoding GetLittleIndianFiles(string srcFile)
{
byte[] b = new byte[5];
using (System.IO.FileStream file = new System.IO.FileStream(srcFile, System.IO.FileMode.Open, System.IO.FileAccess.Read, System.IO.FileShare.Read))
{
int numRead = file.Read(b, 0, 5);
if (numRead < 5)
System.Array.Resize(ref b, numRead);
file.Close();
} // End Using file
if (b.Length >= 2 && b[0] == 0xFF && b[1] == 0xFE)
return System.Text.Encoding.Unicode; // UTF-16, little-endian
return null;
}
}
public class FilePath
{
public string filePath { get; set; }
public string targetFilePath { get; set; }
}
}