CSV - Guess Encoding, Handle Null-string Escape

docs/topics/reading-and-writing-to-file.md

@@ -458,6 +458,24 @@ $reader->setSheetIndex(0);
 
 $spreadsheet = $reader->load("sample.csv");
 ```
+You may also let PhpSpreadsheet attempt to guess the input encoding.
+It will do so based on a test for BOM (UTF-8, UTF-16BE, UTF-16LE, UTF-32BE,
+or UTF-32LE),
+or by doing heuristic tests for those encodings, falling back to a
+specifiable encoding (default is CP1252) if all of those tests fail.
+
+```php
+$reader = new \PhpOffice\PhpSpreadsheet\Reader\Csv();
+$encoding = \PhpOffice\PhpSpreadsheet\Reader\Csv::guessEncoding('sample.csv');
+// or, e.g. $encoding = \PhpOffice\PhpSpreadsheet\Reader\Csv::guessEncoding(
+//                      'sample.csv', 'ISO-8859-2');
+$reader->setInputEncoding($encoding);
+$reader->setDelimiter(';');
+$reader->setEnclosure('');
+$reader->setSheetIndex(0);
+
+$spreadsheet = $reader->load('sample.csv');
+```
 
 #### Read a specific worksheet
 

src/PhpSpreadsheet/Reader/Csv.php

@@ -9,6 +9,21 @@
 
 class Csv extends BaseReader
 {
+    const UTF8_BOM = "\xEF\xBB\xBF";
+    const UTF8_BOM_LEN = 3;
+    const UTF16BE_BOM = "\xfe\xff";
+    const UTF16BE_BOM_LEN = 2;
+    const UTF16BE_LF = "\x00\x0a";
+    const UTF16LE_BOM = "\xff\xfe";
+    const UTF16LE_BOM_LEN = 2;
+    const UTF16LE_LF = "\x0a\x00";
+    const UTF32BE_BOM = "\x00\x00\xfe\xff";
+    const UTF32BE_BOM_LEN = 4;
+    const UTF32BE_LF = "\x00\x00\x00\x0a";
+    const UTF32LE_BOM = "\xff\xfe\x00\x00";
+    const UTF32LE_BOM_LEN = 4;
+    const UTF32LE_LF = "\x0a\x00\x00\x00";
+
     /**
      * Input encoding.
      *
@@ -90,12 +105,8 @@ protected function skipBOM(): void
     {
         rewind($this->fileHandle);
 
-        switch ($this->inputEncoding) {
-            case 'UTF-8':
-                fgets($this->fileHandle, 4) == "\xEF\xBB\xBF" ?
-                    fseek($this->fileHandle, 3) : fseek($this->fileHandle, 0);
-
-                break;
+        if (fgets($this->fileHandle, self::UTF8_BOM_LEN + 1) !== self::UTF8_BOM) {
+            rewind($this->fileHandle);
         }
     }
 
@@ -213,7 +224,9 @@ function ($sum, $value) use ($median) {
     private function getNextLine()
     {
         $line = '';
-        $enclosure = '(?<!' . preg_quote($this->escapeCharacter, '/') . ')' . preg_quote($this->enclosure, '/');
+        $enclosure = ($this->escapeCharacter === '' ? ''
+            : ('(?<!' . preg_quote($this->escapeCharacter, '/') . ')'))
+            . preg_quote($this->enclosure, '/');
 
         do {
             // Get the next line in the file
@@ -307,7 +320,7 @@ private function openFileOrMemory($pFilename): void
             $this->fileHandle = fopen('php://memory', 'r+b');
             $data = StringHelper::convertEncoding($entireFile, 'UTF-8', $this->inputEncoding);
             fwrite($this->fileHandle, $data);
-            rewind($this->fileHandle);
+            $this->skipBOM();
         }
     }
 
@@ -531,4 +544,63 @@ public function canRead($pFilename)
 
         return in_array($type, $supportedTypes, true);
     }
+
+    private static function guessEncodingTestNoBom(string &$encoding, string &$contents, string $compare, string $setEncoding): void
+    {
+        if ($encoding === '') {
+            $pos = strpos($contents, $compare);
+            if ($pos !== false && $pos % strlen($compare) === 0) {
+                $encoding = $setEncoding;
+            }
+        }
+    }
+
+    private static function guessEncodingNoBom(string $filename): string
+    {
+        $encoding = '';
+        $contents = file_get_contents($filename);
+        self::guessEncodingTestNoBom($encoding, $contents, self::UTF32BE_LF, 'UTF-32BE');
+        self::guessEncodingTestNoBom($encoding, $contents, self::UTF32LE_LF, 'UTF-32LE');
+        self::guessEncodingTestNoBom($encoding, $contents, self::UTF16BE_LF, 'UTF-16BE');
+        self::guessEncodingTestNoBom($encoding, $contents, self::UTF16LE_LF, 'UTF-16LE');
+        if ($encoding === '' && preg_match('//u', $contents) === 1) {
+            $encoding = 'UTF-8';
+        }
+
+        return $encoding;
+    }
+
+    private static function guessEncodingTestBom(string &$encoding, string $first4, string $compare, string $setEncoding): void
+    {
+        if ($encoding === '') {
+            if ($compare === substr($first4, 0, strlen($compare))) {
+                $encoding = $setEncoding;
+            }
+        }
+    }
+
+    private static function guessEncodingBom(string $filename): string
+    {
+        $encoding = '';
+        $first4 = file_get_contents($filename, false, null, 0, 4);
+        if ($first4 !== false) {
+            self::guessEncodingTestBom($encoding, $first4, self::UTF8_BOM, 'UTF-8');
+            self::guessEncodingTestBom($encoding, $first4, self::UTF16BE_BOM, 'UTF-16BE');
+            self::guessEncodingTestBom($encoding, $first4, self::UTF32BE_BOM, 'UTF-32BE');
+            self::guessEncodingTestBom($encoding, $first4, self::UTF32LE_BOM, 'UTF-32LE');
+            self::guessEncodingTestBom($encoding, $first4, self::UTF16LE_BOM, 'UTF-16LE');
+        }
+
+        return $encoding;
+    }
+
+    public static function guessEncoding(string $filename, string $dflt = 'CP1252'): string
+    {
+        $encoding = self::guessEncodingBom($filename);
+        if ($encoding === '') {
+            $encoding = self::guessEncodingNoBom($filename);
+        }
+
+        return ($encoding === '') ? $dflt : $encoding;
+    }
 }

tests/PhpSpreadsheetTests/Reader/CsvTest.php

@@ -275,4 +275,66 @@ public function testReadNonexistentFileName(): void
         $reader = new Csv();
         $reader->load('tests/data/Reader/CSV/encoding.utf8.csvxxx');
     }
+
+    /**
+     * @dataProvider providerEscapes
+     */
+    public function testInferSeparator(string $escape, string $delimiter): void
+    {
+        $reader = new Csv();
+        $reader->setEscapeCharacter($escape);
+        $filename = 'tests/data/Reader/CSV/escape.csv';
+        $reader->listWorksheetInfo($filename);
+        self::assertEquals($delimiter, $reader->getDelimiter());
+    }
+
+    public function providerEscapes()
+    {
+        return [
+            ['\\', ';'],
+            ["\x0", ','],
+            [(version_compare(PHP_VERSION, '7.4') < 0) ? "\x0" : '', ','],
+        ];
+    }
+
+    /**
+     * @dataProvider providerGuessEncoding
+     */
+    public function testGuessEncoding(string $filename): void
+    {
+        $reader = new Csv();
+        $reader->setInputEncoding(Csv::guessEncoding($filename));
+        $spreadsheet = $reader->load($filename);
+        $sheet = $spreadsheet->getActiveSheet();
+        self::assertEquals('première', $sheet->getCell('A1')->getValue());
+        self::assertEquals('sixième', $sheet->getCell('C2')->getValue());
+    }
+
+    public function providerGuessEncoding()
+    {
+        return [
+            ['tests/data/Reader/CSV/premiere.utf8.csv'],
+            ['tests/data/Reader/CSV/premiere.utf8bom.csv'],
+            ['tests/data/Reader/CSV/premiere.utf16be.csv'],
+            ['tests/data/Reader/CSV/premiere.utf16bebom.csv'],
+            ['tests/data/Reader/CSV/premiere.utf16le.csv'],
+            ['tests/data/Reader/CSV/premiere.utf16lebom.csv'],
+            ['tests/data/Reader/CSV/premiere.utf32be.csv'],
+            ['tests/data/Reader/CSV/premiere.utf32bebom.csv'],
+            ['tests/data/Reader/CSV/premiere.utf32le.csv'],
+            ['tests/data/Reader/CSV/premiere.utf32lebom.csv'],
+            ['tests/data/Reader/CSV/premiere.win1252.csv'],
+        ];
+    }
+
+    public function testGuessEncodingDefltIso2(): void
+    {
+        $filename = 'tests/data/Reader/CSV/premiere.win1252.csv';
+        $reader = new Csv();
+        $reader->setInputEncoding(Csv::guessEncoding($filename, 'ISO-8859-2'));
+        $spreadsheet = $reader->load($filename);
+        $sheet = $spreadsheet->getActiveSheet();
+        self::assertEquals('premičre', $sheet->getCell('A1')->getValue());
+        self::assertEquals('sixičme', $sheet->getCell('C2')->getValue());
+    }
 }

tests/data/Reader/CSV/escape.csv

@@ -0,0 +1,4 @@
+a\"hello;hello;hello;\",b\"hello;hello;hello;\",c\"\hello;hello;hello;\"
+a\"hello;hello;hello;\",b\"hello;hello;hello;\",c\"\hello;hello;hello;\",d
+a\"hello;hello;hello;\",b\"hello;hello;hello;\",c\"\hello;hello;hello;\"
+a\"hello;hello;hello;\",b\"hello;hello;hello;\",c\"\hello;hello;hello;\"

tests/data/Reader/CSV/premiere.utf8.csv

@@ -0,0 +1,2 @@
+première,second,troisième
+Quatrième,cinquième,sixième

tests/data/Reader/CSV/premiere.utf8bom.csv

@@ -0,0 +1,2 @@
+première,second,troisième
+Quatrième,cinquième,sixième

tests/data/Reader/CSV/premiere.win1252.csv

@@ -0,0 +1,2 @@
+premi�re,second,troisi�me
+Quatri�me,cinqui�me,sixi�me