Refactor IDNA

make_idna_table.py

@@ -21,76 +21,55 @@ print('''\
 // except according to those terms.
 
 // Generated by make_idna_table.py
-''')
 
-print('''\
-#[allow(non_camel_case_types)]
-pub enum MappingStatus {
-    valid,
-    ignored,
-    mapped,
-    deviation,
-    disallowed,
-    disallowed_STD3_valid,
-    disallowed_STD3_mapped,
+pub enum Mapping {
+    Valid,
+    Ignored,
+    Mapped(&'static str),
+    Deviation(&'static str),
+    Disallowed,
+    DisallowedStd3Valid,
+    DisallowedStd3Mapped(&'static str),
 }
 
-pub struct Mapping {
+pub struct Range {
     pub from: char,
     pub to: char,
-    pub status: MappingStatus,
-    pub mapping: &'static [char],
+    pub mapping: Mapping,
 }
 
-''')
-
-print("static NONE: [char;0] = [];")
+use self::Mapping::*;
 
-txt = open("IdnaMappingTable.txt")
-line_no = 0
-
-for line in txt:
-    # remove comments
-    head, sep, tail = line.partition('#')
-    # skip empty lines
-    if len(head.strip()) == 0:
-        continue
-    line_no = line_no + 1
+pub static TABLE: &'static [Range] = &[
+''')
 
 txt = open("IdnaMappingTable.txt")
-print("pub static TABLE: &'static [Mapping] = &[")
 
 def char(s):
-    return "'%s'" % unichr(int(s, 16)).replace('\\', '\\\\').replace('\'', '\\\'').encode('utf8')
-
-mappings = []
+    return (unichr(int(s, 16))
+        .encode('utf8')
+        .replace('\\', '\\\\')
+        .replace('"', '\\"')
+        .replace('\0', '\\0'))
 
 for line in txt:
     # remove comments
-    head, sep, tail = line.partition('#')
+    line, _, _ = line.partition('#')
     # skip empty lines
-    if len(head.strip()) == 0:
+    if len(line.strip()) == 0:
         continue
-    table_line = head.split(';')
-    if table_line[0].strip() == 'D800..DFFF':
+    fields = line.split(';')
+    if fields[0].strip() == 'D800..DFFF':
         continue  # Surrogates don't occur in Rust strings.
-    first, sep, last = table_line[0].strip().partition('..')
-    if len(last)==0:
+    first, _, last = fields[0].strip().partition('..')
+    if not last:
         last = first
-    mapping = "NONE"
-    if len(table_line)>2:
-        if len(table_line[2].strip())>0:
-            codes = table_line[2].strip().split(' ')
-            newmap = ""
-            for code in codes:
-                newmap = newmap + char(code) + ", "
-            newmap = "[" + newmap + "]"
-            mapping = "MAPPING_%s_%s" % (first, last)
-            static_array = "static %s : [char; %d] = %s;" % (mapping, len(codes), newmap)
-            mappings.append(static_array)
-    print "    Mapping{ from: %s, to: %s, status: MappingStatus::%s, mapping: &%s }," % (char(first), char(last), table_line[1].strip(), mapping)
+    mapping = fields[1].strip().replace('_', ' ').title().replace(' ', '')
+    if len(fields) > 2:
+        if fields[2].strip():
+            mapping += '("%s")' % ''.join(char(c) for c in fields[2].strip().split(' '))
+        elif mapping == "Deviation":
+            mapping += '("")'
+    print("    Range { from: '%s', to: '%s', mapping: %s }," % (char(first), char(last), mapping))
 
 print("];")
-
-for mapping in mappings:
-    print mapping

src/idna.rs

@@ -2,41 +2,12 @@
 //!
 //! https://url.spec.whatwg.org/#idna
 
-use idna_mapping::*;
+use idna_mapping::{TABLE, Mapping};
 use punycode;
 use std::ascii::AsciiExt;
 use unicode_normalization::UnicodeNormalization;
 
-fn idna_mapped(mapping: &'static [char]) -> Result<String, Error> {
-    let mut ret = "".to_string();
-    for &c in mapping {
-        ret.push(c)
-    }
-    return Ok(ret);
-}
-
-fn idna_deviation(codepoint: char, mapping: &'static [char], transitional: bool) -> Result<String, Error> {
-    if transitional {
-       return idna_mapped(mapping);
-    }
-    return Ok(codepoint.to_string());
-}
-
-fn idna_disallowed_std3_valid(codepoint: char, use_std3_asciirules: bool) -> Result<String, Error> {
-    if use_std3_asciirules {
-        return Err(Error::DissallowedByStd3AsciiRules);
-    }
-    return Ok(codepoint.to_string());
-}
-
-fn idna_disallowed_std3_mapped(mapping: &'static [char], use_std3_asciirules: bool) -> Result<String, Error> {
-    if use_std3_asciirules {
-        return Err(Error::DissallowedMappedInStd3);
-    }
-    return idna_mapped(mapping);
-}
-
-fn map_char(codepoint: char, flags: Uts46Flags) -> Result<String, Error> {
+fn map_char(codepoint: char, flags: Uts46Flags, output: &mut String) -> Result<(), Error> {
     let mut min = 0;
     let mut max = TABLE.len() - 1;
     while max > min {
@@ -51,29 +22,51 @@ fn map_char(codepoint: char, flags: Uts46Flags) -> Result<String, Error> {
         }
     }
 
-    let mapping = TABLE[min].mapping;
-
-    match TABLE[min].status {
-        MappingStatus::valid => Ok(codepoint.to_string()),
-        MappingStatus::ignored => Ok("".to_string()),
-        MappingStatus::mapped => idna_mapped(mapping),
-        MappingStatus::deviation => {
-            idna_deviation(codepoint, mapping, flags.transitional_processing)
+    match TABLE[min].mapping {
+        Mapping::Valid => output.push(codepoint),
+        Mapping::Ignored => {},
+        Mapping::Mapped(mapping) => output.push_str(mapping),
+        Mapping::Deviation(mapping) => {
+            if flags.transitional_processing {
+                output.push_str(mapping)
+            } else {
+                output.push(codepoint)
+            }
         }
-        MappingStatus::disallowed => Err(Error::DissallowedCharacter),
-        MappingStatus::disallowed_STD3_valid => {
-            idna_disallowed_std3_valid(codepoint, flags.use_std3_ascii_rules)
+        Mapping::Disallowed => return Err(Error::DissallowedCharacter),
+        Mapping::DisallowedStd3Valid => {
+            if flags.use_std3_ascii_rules {
+                return Err(Error::DissallowedByStd3AsciiRules);
+            } else {
+                output.push(codepoint)
+            }
         }
-        MappingStatus::disallowed_STD3_mapped => {
-            idna_disallowed_std3_mapped(mapping, flags.use_std3_ascii_rules)
+        Mapping::DisallowedStd3Mapped(mapping) => {
+            if flags.use_std3_ascii_rules {
+                return Err(Error::DissallowedMappedInStd3);
+            } else {
+                output.push_str(mapping)
+            }
         }
     }
+    Ok(())
+}
+
+/// http://www.unicode.org/reports/tr46/#Processing
+fn uts46_processing(domain: &str, flags: Uts46Flags) -> Result<String, Error> {
+    let mut mapped = String::new();
+    for c in domain.chars() {
+        try!(map_char(c, flags, &mut mapped))
+    }
+    Ok(mapped.nfc().collect())
+    // FIXME: steps 3 & 4: Break & Convert/Validate
 }
 
 #[derive(Copy, Clone)]
 pub struct Uts46Flags {
    pub use_std3_ascii_rules: bool,
    pub transitional_processing: bool,
+   // FIXME: verify_dns_length: bool,
 }
 
 pub enum Error {
@@ -85,21 +78,8 @@ pub enum Error {
 
 /// http://www.unicode.org/reports/tr46/#ToASCII
 pub fn uts46_to_ascii(domain: &str, flags: Uts46Flags) -> Result<String, Error> {
-    let mut ret = String::new();
-    for c in domain.chars() {
-        match map_char(c, flags) {
-            Ok(mystr) => ret.push_str(&mystr),
-            Err(x) => return Err(x)
-        }
-    }
-
-    // normalize NFC
-    let ret = ret.nfc().collect::<String>();
-
-    let vec: Vec<&str> = ret.split(".").collect();
     let mut result = String::new();
-
-    for label in vec {
+    for label in try!(uts46_processing(domain, flags)).split(".") {
         if label.is_ascii() {
             if result.len() > 0 {
                 result.push('.');
@@ -118,6 +98,7 @@ pub fn uts46_to_ascii(domain: &str, flags: Uts46Flags) -> Result<String, Error>
             }
         }
     }
+    // FIXME: step 4: optionally verify dns length
 
     return Ok(result);
 }
@@ -127,5 +108,6 @@ pub fn domain_to_ascii(domain: &str) -> Result<String, Error> {
     uts46_to_ascii(domain, Uts46Flags {
         use_std3_ascii_rules: false,
         transitional_processing: true,
+        //verify_dns_length: false,
     })
 }

tests/idna.rs

@@ -24,12 +24,12 @@ fn test_uts46() {
 
         let mut pieces = line.split(';').map(|x| x.trim()).collect::<Vec<&str>>();
 
-        let testType = pieces.remove(0);
+        let test_type = pieces.remove(0);
         let original = pieces.remove(0);
         let source = unescape(original);
-        let toUnicode = pieces.remove(0);
-        let toAscii = pieces.remove(0);
-        let nv8 = pieces.len() > 0;
+        let to_unicode = pieces.remove(0);
+        let to_ascii = pieces.remove(0);
+        let _nv8 = pieces.len() > 0;
 
         if expected_failure {
             continue;
@@ -37,20 +37,20 @@ fn test_uts46() {
 
         let result = idna::uts46_to_ascii(&source, idna::Uts46Flags {
             use_std3_ascii_rules: true,
-            transitional_processing: testType != "N"
+            transitional_processing: test_type != "N"
         });
         let res = result.ok();
 
-        if toAscii.starts_with("[") {
+        if to_ascii.starts_with("[") {
             //assert!(res == None, "Expected error. result: {} | original: {} | source: {}", res.unwrap(), original, source);
             continue;
         }
 
-        let toAscii = if toAscii.len() > 0 {
-            toAscii.to_string()
+        let to_ascii = if to_ascii.len() > 0 {
+            to_ascii.to_string()
         } else {
-            if toUnicode.len() > 0 {
-                toUnicode.to_string()
+            if to_unicode.len() > 0 {
+                to_unicode.to_string()
             } else {
                 source.clone()
             }
@@ -58,7 +58,7 @@ fn test_uts46() {
 
         assert!(res != None, "Couldn't parse {} ", source);
         let output = res.unwrap();
-        assert!(output == toAscii, "result: {} | expected: {} | original: {} | source: {}", output, toAscii, original, source);
+        assert!(output == to_ascii, "result: {} | expected: {} | original: {} | source: {}", output, to_ascii, original, source);
     }
 }