Align mime type parser to spec

The most notable two changes are:
a) the parser now uses html whitespace instead of ascii whitespace
b) the parser now correctly handles empty quoted values

The commit additionally fixes a bug in the parser that would fail
parsing if an invalid name was encountered and its corresponding value
contained a quoted semicolon.

data-url/src/mime.rs

@@ -35,13 +35,13 @@ impl FromStr for Mime {
 }
 
 fn parse(s: &str) -> Option<Mime> {
-    let trimmed = s.trim_matches(ascii_whitespace);
+    let trimmed = s.trim_matches(http_whitespace);
 
     let (type_, rest) = split2(trimmed, '/');
     require!(only_http_token_code_points(type_) && !type_.is_empty());
 
     let (subtype, rest) = split2(rest?, ';');
-    let subtype = subtype.trim_end_matches(ascii_whitespace);
+    let subtype = subtype.trim_end_matches(http_whitespace);
     require!(only_http_token_code_points(subtype) && !subtype.is_empty());
 
     let mut parameters = Vec::new();
@@ -66,11 +66,12 @@ fn parse_parameters(s: &str, parameters: &mut Vec<(String, String)>) {
     let mut semicolon_separated = s.split(';');
 
     while let Some(piece) = semicolon_separated.next() {
-        let piece = piece.trim_start_matches(ascii_whitespace);
+        let piece = piece.trim_start_matches(http_whitespace);
         let (name, value) = split2(piece, '=');
-        if name.is_empty() || !only_http_token_code_points(name) || contains(parameters, name) {
-            continue;
-        }
+        // We can not early return on an invalid name here, because the value
+        // parsing later may consume more semicolon seperated pieces.
+        let name_valid =
+            !name.is_empty() && only_http_token_code_points(name) && !contains(parameters, name);
         if let Some(value) = value {
             let value = if let Some(stripped) = value.strip_prefix('"') {
                 let max_len = stripped.len().saturating_sub(1); // without end quote
@@ -80,7 +81,17 @@ fn parse_parameters(s: &str, parameters: &mut Vec<(String, String)>) {
                     while let Some(c) = chars.next() {
                         match c {
                             '"' => break 'until_closing_quote,
-                            '\\' => unescaped_value.push(chars.next().unwrap_or('\\')),
+                            '\\' => unescaped_value.push(chars.next().unwrap_or_else(|| {
+                                semicolon_separated
+                                    .next()
+                                    .map(|piece| {
+                                        // A semicolon inside a quoted value is not a separator
+                                        // for the next parameter, but part of the value.
+                                        chars = piece.chars();
+                                        ';'
+                                    })
+                                    .unwrap_or('\\')
+                            })),
                             _ => unescaped_value.push(c),
                         }
                     }
@@ -93,13 +104,16 @@ fn parse_parameters(s: &str, parameters: &mut Vec<(String, String)>) {
                         break;
                     }
                 }
-                if !valid_value(&unescaped_value) {
+                if !name_valid || !valid_value(value) {
                     continue;
                 }
                 unescaped_value
             } else {
-                let value = value.trim_end_matches(ascii_whitespace);
-                if !valid_value(value) {
+                let value = value.trim_end_matches(http_whitespace);
+                if value.is_empty() {
+                    continue;
+                }
+                if !name_valid || !valid_value(value) {
                     continue;
                 }
                 value.to_owned()
@@ -117,7 +131,7 @@ fn valid_value(s: &str) -> bool {
     s.chars().all(|c| {
         // <https://mimesniff.spec.whatwg.org/#http-quoted-string-token-code-point>
         matches!(c, '\t' | ' '..='~' | '\u{80}'..='\u{FF}')
-    }) && !s.is_empty()
+    })
 }
 
 /// <https://mimesniff.spec.whatwg.org/#serializing-a-mime-type>
@@ -130,7 +144,7 @@ impl fmt::Display for Mime {
             f.write_str(";")?;
             f.write_str(name)?;
             f.write_str("=")?;
-            if only_http_token_code_points(value) {
+            if only_http_token_code_points(value) && !value.is_empty() {
                 f.write_str(value)?
             } else {
                 f.write_str("\"")?;
@@ -147,8 +161,8 @@ impl fmt::Display for Mime {
     }
 }
 
-fn ascii_whitespace(c: char) -> bool {
-    matches!(c, ' ' | '\t' | '\n' | '\r' | '\x0C')
+fn http_whitespace(c: char) -> bool {
+    matches!(c, ' ' | '\t' | '\n' | '\r')
 }
 
 fn only_http_token_code_points(s: &str) -> bool {

data-url/tests/base64.json

@@ -49,6 +49,9 @@
   ["ab=c=", null],
   ["abc=d", null],
   ["abc=d=", null],
+  ["ab\u000Bcd", null],
+  ["ab\u3000cd", null],
+  ["ab\u3001cd", null],
   ["ab\tcd", [105, 183, 29]],
   ["ab\ncd", [105, 183, 29]],
   ["ab\fcd", [105, 183, 29]],

data-url/tests/data-urls.json

@@ -52,6 +52,12 @@
   ["data:text/plain;Charset=UTF-8,%C2%B1",
    "text/plain;charset=UTF-8",
    [194, 177]],
+  ["data:text/plain;charset=windows-1252,áñçə💩",
+   "text/plain;charset=windows-1252",
+   [195, 161, 195, 177, 195, 167, 201, 153, 240, 159, 146, 169]],
+  ["data:text/plain;charset=UTF-8,áñçə💩",
+   "text/plain;charset=UTF-8",
+   [195, 161, 195, 177, 195, 167, 201, 153, 240, 159, 146, 169]],
   ["data:image/gif,%C2%B1",
    "image/gif",
    [194, 177]],
@@ -100,14 +106,23 @@
   ["data:image/png,X X",
    "image/png",
    [88, 32, 88]],
+  ["data:application/javascript,X X",
+   "application/javascript",
+   [88, 32, 88]],
   ["data:application/xml,X X",
    "application/xml",
    [88, 32, 88]],
+  ["data:text/javascript,X X",
+   "text/javascript",
+   [88, 32, 88]],
+  ["data:text/plain,X X",
+   "text/plain",
+   [88, 32, 88]],
   ["data:unknown/unknown,X X",
    "unknown/unknown",
    [88, 32, 88]],
   ["data:text/plain;a=\",\",X",
-   "text/plain",
+   "text/plain;a=\"\"",
    [34, 44, 88]],
   ["data:text/plain;a=%2C,X",
    "text/plain;a=%2C",

data-url/tests/mime-types.json

@@ -32,6 +32,12 @@
     "navigable": true,
     "encoding": "GBK"
   },
+  {
+    "input": "text/html;charset=();charset=GBK",
+    "output": "text/html;charset=\"()\"",
+    "navigable": true,
+    "encoding": null
+  },
   "Spaces",
   {
     "input": "text/html;charset =gbk",
@@ -57,6 +63,37 @@
     "navigable": true,
     "encoding": "GBK"
   },
+  {
+    "input": "text/html;charset= \"gbk\"",
+    "output": "text/html;charset=\" \\\"gbk\\\"\"",
+    "navigable": true,
+    "encoding": null
+  },
+  "0x0B and 0x0C",
+  {
+    "input": "text/html;charset=\u000Bgbk",
+    "output": "text/html",
+    "navigable": true,
+    "encoding": null
+  },
+  {
+    "input": "text/html;charset=\u000Cgbk",
+    "output": "text/html",
+    "navigable": true,
+    "encoding": null
+  },
+  {
+    "input": "text/html;\u000Bcharset=gbk",
+    "output": "text/html",
+    "navigable": true,
+    "encoding": null
+  },
+  {
+    "input": "text/html;\u000Ccharset=gbk",
+    "output": "text/html",
+    "navigable": true,
+    "encoding": null
+  },
   "Single quotes are a token, not a delimiter",
   {
     "input": "text/html;charset='gbk'",
@@ -76,6 +113,12 @@
     "navigable": true,
     "encoding": null
   },
+  {
+    "input": "text/html;charset=';charset=GBK",
+    "output": "text/html;charset='",
+    "navigable": true,
+    "encoding": null
+  },
   "Invalid parameters",
   {
     "input": "text/html;test;charset=gbk",
@@ -113,6 +156,18 @@
     "navigable": true,
     "encoding": "GBK"
   },
+  {
+    "input": "text/html;charset= \"\u007F;charset=GBK",
+    "output": "text/html;charset=GBK",
+    "navigable": true,
+    "encoding": "GBK"
+  },
+  {
+    "input": "text/html;charset=\"\u007F;charset=foo\";charset=GBK",
+    "output": "text/html;charset=GBK",
+    "navigable": true,
+    "encoding": "GBK"
+  },
   "Double quotes",
   {
     "input": "text/html;charset=\"gbk\"",
@@ -138,6 +193,12 @@
     "navigable": true,
     "encoding": "GBK"
   },
+  {
+    "input": "text/html;charset=\"gbk \"",
+    "output": "text/html;charset=\"gbk \"",
+    "navigable": true,
+    "encoding": "GBK"
+  },
   {
     "input": "text/html;charset=\"\\ gbk\"",
     "output": "text/html;charset=\" gbk\"",
@@ -156,6 +217,18 @@
     "navigable": true,
     "encoding": "GBK"
   },
+  {
+    "input": "text/html;charset=\"\";charset=GBK",
+    "output": "text/html;charset=\"\"",
+    "navigable": true,
+    "encoding": null
+  },
+  {
+    "input": "text/html;charset=\";charset=GBK",
+    "output": "text/html;charset=\";charset=GBK\"",
+    "navigable": true,
+    "encoding": null
+  },
   "Unexpected code points",
   {
     "input": "text/html;charset={gbk}",
@@ -175,6 +248,20 @@
     "input": "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789/0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789",
     "output": "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789/0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789"
   },
+  "Invalid names",
+  {
+    "input": "text/html;a]=bar;b[=bar;c=bar",
+    "output": "text/html;c=bar"
+  },
+  "Semicolons in value",
+  {
+    "input": "text/html;valid=\";\";foo=bar",
+    "output": "text/html;valid=\";\";foo=bar"
+  },
+  {
+    "input": "text/html;in]valid=\";asd=foo\";foo=bar",
+    "output": "text/html;foo=bar"
+  },
   "Valid",
   {
     "input": "!#$%&'*+-.^_`|~0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz/!#$%&'*+-.^_`|~0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz;!#$%&'*+-.^_`|~0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz=!#$%&'*+-.^_`|~0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz",
@@ -202,6 +289,18 @@
     "input": "x/x;x=\t",
     "output": "x/x"
   },
+  {
+    "input": "x/x\n\r\t ;x=x",
+    "output": "x/x;x=x"
+  },
+  {
+    "input": "\n\r\t x/x;x=x\n\r\t ",
+    "output": "x/x;x=x"
+  },
+  {
+    "input": "x/x;\n\r\t x=x\n\r\t ;x=y",
+    "output": "x/x;x=x"
+  },
   "Latin1",
   {
     "input": "text/html;test=\u00FF;charset=gbk",
@@ -215,6 +314,22 @@
     "output": "x/x;x=x"
   },
   "Failure",
+  {
+    "input": "\u000Bx/x",
+    "output": null
+  },
+  {
+    "input": "\u000Cx/x",
+    "output": null
+  },
+  {
+    "input": "x/x\u000B",
+    "output": null
+  },
+  {
+    "input": "x/x\u000C",
+    "output": null
+  },
   {
     "input": "",
     "output": null
@@ -223,6 +338,10 @@
     "input": "\t",
     "output": null
   },
+  {
+    "input": "/",
+    "output": null
+  },
   {
     "input": "bogus",
     "output": null
@@ -247,6 +366,10 @@
     "input": "(/)",
     "output": null
   },
+  {
+    "input": "ÿ/ÿ",
+    "output": null
+  },
   {
     "input": "text/html(;doesnot=matter",
     "output": null
@@ -258,5 +381,17 @@
   {
     "input": "\u0100/\u0100",
     "output": null
+  },
+  {
+    "input": "text /html",
+    "output": null
+  },
+  {
+    "input": "text/ html",
+    "output": null
+  },
+  {
+    "input": "\"text/html\"",
+    "output": null
   }
 ]