Detect the encoding/codepage of a text file

2023-12-13 UPDATE: Please notice that there is a bug in the file TextFileEncodingDetector.cs as commented here.

In Method DetectSuspiciousUTF8SequenceLength instead of:

SampleBytes.Length >= currentPos + 1

it should be:

SampleBytes.Length > currentPos + 1

One my example on how to detect the encoding/codepage of a text file using TextFileEncodingDetector project in .NET core.

First to mention that not even Notepad++ can't detect the encoding/codepage of a text file correctly. Try to save one file with, for example, Windows-1252 character set and reopen it again.
In my case I will save few files with different encoding in C#. In order to test if files were correctly saved I have opened them with binary editor in Visual studio 2019:

File -> Open

Open with

Binary editor

Check hexadecimal value of unicode code points for example UTF-16 Table, UTF-8 Table, or for example "ß" - German eszett, here or here.

And here is da code:

using KlerksSoft;
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Text;

namespace EncodingDetector
{
    class Program
    {
        static void Main(string[] args)
        {
            Console.WriteLine("Read all encodings from the system and write file for each encoding.");

            string exampleStringToWrite = "üöäß;ÜÖÄß@€µ";

            List<string> savedFiles = new List<string>();
            ProcessModule processModule = Process.GetCurrentProcess().MainModule;
            string exePath = string.Empty;
            if (processModule != null)
            {
                exePath = Path.GetDirectoryName(processModule.FileName);
            }

            foreach (EncodingInfo encodingInfo in Encoding.GetEncodings())
            {
                string fileName = $"{encodingInfo.DisplayName}.txt";
                foreach (char c in Path.GetInvalidFileNameChars())
                {
                    fileName = fileName.Replace(c, '_');
                }

                fileName = Path.Combine(exePath ?? string.Empty, fileName);

                using (StreamWriter sw = new StreamWriter(File.Open(fileName, FileMode.Create), encodingInfo.GetEncoding()))
                {
                    sw.WriteLine(exampleStringToWrite);
                    savedFiles.Add(fileName);
                    Console.WriteLine($"File {fileName} saved.");
                }
            }

            string windows1252File = Path.Combine(exePath ?? string.Empty, "windows1252.txt");
            Encoding.RegisterProvider(CodePagesEncodingProvider.Instance);
            using (StreamWriter sw = new StreamWriter(File.Open(windows1252File, FileMode.Create), Encoding.GetEncoding(1252)))
            {
                sw.WriteLine(exampleStringToWrite);
                Console.WriteLine($"File {windows1252File} saved.");
            }


            Console.WriteLine("Press any key to detect encodings");
            Console.ReadKey();

            foreach (string savedFile in savedFiles)
            {
                DisplayEncodingInConsole(savedFile);
            }

            DisplayEncodingInConsole(windows1252File);

            Console.WriteLine("Finished");
            Console.ReadKey();
        }

        private static void DisplayEncodingInConsole(string fileName)
        {
            Encoding encoding = TextFileEncodingDetector.DetectTextFileEncoding(fileName);

            if (encoding is null)
            {
                Console.WriteLine($"File {fileName} is most probably encoded as {Encoding.Default}");
            }
            else
            {
                Console.WriteLine($"File {fileName} encoded as {encoding}");
            }
        }
    }
}

Here is the source code.