bug #10983 [DomCrawler] Fixed charset detection in html5 meta charset tag (77web)

fabpot · fabpot · commit cff410507f84 · 2014-05-27T00:15:18.000+02:00
This PR was squashed before being merged into the 2.3 branch (closes #10983). Discussion ---------- [DomCrawler] Fixed charset detection in html5 meta charset tag | Q | A | ------------- | --- | Bug fix? | yes | New feature? | no | BC breaks? | no | Deprecations? | no | Tests pass? | yes | Fixed tickets | N/A | License | MIT It may be minor to folks with ascii-charactered language, but is critical for us Japanese. Many Japanese websites with SJIS encoding have "Shift_JIS" as their encoding declaration. Commits ------- 172e752 [DomCrawler] Fixed charset detection in html5 meta charset tag
diff --git a/src/Symfony/Component/DomCrawler/Crawler.php b/src/Symfony/Component/DomCrawler/Crawler.php
@@ -108,8 +108,10 @@ public function addContent($content, $type = null)
             }
         }
 
+        // http://www.w3.org/TR/encoding/#encodings
+        // http://www.w3.org/TR/REC-xml/#NT-EncName
         if (null === $charset &&
-            preg_match('/\<meta[^\>]+charset *= *["\']?([a-zA-Z\-0-9]+)/i', $content, $matches)) {
+            preg_match('/\<meta[^\>]+charset *= *["\']?([a-zA-Z\-0-9_:.]+)/i', $content, $matches)) {
             $charset = $matches[1];
         }
 
diff --git a/src/Symfony/Component/DomCrawler/Tests/CrawlerTest.php b/src/Symfony/Component/DomCrawler/Tests/CrawlerTest.php
@@ -232,6 +232,10 @@ public function testAddContent()
         $crawler = new Crawler();
         $crawler->addContent('<html><meta http-equiv="Content-Type" content="text/html; charset=utf-8" /><span>中文</span></html>');
         $this->assertEquals('中文', $crawler->filterXPath('//span')->text(), '->addContent() guess wrong charset');
+
+        $crawler = new Crawler();
+        $crawler->addContent(mb_convert_encoding('<html><head><meta charset="Shift_JIS"></head><body>日本語</body></html>', 'SJIS', 'UTF-8'));
+        $this->assertEquals('日本語', $crawler->filterXPath('//body')->text(), '->addContent() can recognize "Shift_JIS" in html5 meta charset tag');
     }
 
     /**

Original file line number	Diff line number	Diff line change
`@@ -108,8 +108,10 @@ public function addContent($content, $type = null)`
`108`	`108`	`}`
`109`	`109`	`}`
`110`	`110`
	`111`	`+ // http://www.w3.org/TR/encoding/#encodings`
	`112`	`+ // http://www.w3.org/TR/REC-xml/#NT-EncName`
`111`	`113`	`if (null === $charset &&`
`112`		`- preg_match('/\<meta[^\>]+charset = ["\']?([a-zA-Z\-0-9]+)/i', $content, $matches)) {`
	`114`	`+ preg_match('/\<meta[^\>]+charset = ["\']?([a-zA-Z\-0-9_:.]+)/i', $content, $matches)) {`
`113`	`115`	`$charset = $matches[1];`
`114`	`116`	`}`
`115`	`117`