graphviz_pure_core/
charset.rs

1//! Byte-faithful strings and the graph `charset` (`lib/common/input.c`, `utils.c`).
2//!
3//! C works on the bytes of its input: attribute values, names and the text of
4//! `-Tdot`/`-Tsvg`/... output are whatever bytes the file held, and only label text
5//! is converted to UTF-8 (`latin1ToUTF8` for `charset=latin1`, `htmlEntityUTF8`
6//! otherwise, whose invalid-UTF-8 recovery treats the offending byte as Latin-1).
7//!
8//! The port keeps strings as `String`. Input that is not valid UTF-8 is decoded
9//! with [`decode`]: valid UTF-8 sequences stay as they are and every other byte
10//! `b` becomes the private-use code point `U+10FF00 + b` (a "raw byte"). [`encode`]
11//! maps them back, so the CLI writes C's exact bytes, and the C conversions run on
12//! [`encode`]d bytes, so `charset=latin1` text is converted exactly once, where C
13//! converts it. Text measurement (which C does byte by byte) also uses the encoded
14//! bytes. Input that genuinely contains `U+10FF00..=U+10FFFF` is not distinguished
15//! from raw bytes (no real-world encoding assigns these code points).
16
17use std::borrow::Cow;
18use std::cell::{Cell, RefCell};
19use std::io::Write;
20
21/// First code point of the raw-byte range (`U+10FF00 + byte`).
22const RAW_BASE: u32 = 0x10_FF00;
23
24/// Decodes input bytes: valid UTF-8 is kept, every other byte becomes a raw-byte char.
25pub fn decode(bytes: Vec<u8>) -> String {
26    match String::from_utf8(bytes) {
27        Ok(s) => s,
28        Err(e) => decode_slice(e.as_bytes()),
29    }
30}
31
32/// [`decode`] for a borrowed slice.
33pub fn decode_slice(bytes: &[u8]) -> String {
34    if let Ok(s) = std::str::from_utf8(bytes) {
35        return s.to_string();
36    }
37    let mut out = String::with_capacity(bytes.len() + 8);
38    for chunk in bytes.utf8_chunks() {
39        out.push_str(chunk.valid());
40        for &b in chunk.invalid() {
41            out.push(raw_char(b));
42        }
43    }
44    out
45}
46
47/// The char that stands for the raw (non-UTF-8) byte `b`; [`encode`] turns it back.
48pub fn raw_char(b: u8) -> char {
49    char::from_u32(RAW_BASE + u32::from(b)).expect("raw byte code point")
50}
51
52/// Whether `s` holds raw-byte chars (non-UTF-8 input bytes).
53pub fn has_raw(s: &str) -> bool {
54    // U+10FFxx is encoded as F4 8F Bx xx.
55    s.as_bytes().contains(&0xF4) && s.chars().any(|c| u32::from(c) >= RAW_BASE)
56}
57
58/// The bytes C would hold for `s`: raw-byte chars become their byte again.
59pub fn encode(s: &str) -> Cow<'_, [u8]> {
60    if !has_raw(s) {
61        return Cow::Borrowed(s.as_bytes());
62    }
63    let mut out = Vec::with_capacity(s.len());
64    let mut buf = [0u8; 4];
65    for c in s.chars() {
66        let v = u32::from(c);
67        if v >= RAW_BASE {
68            out.push((v - RAW_BASE) as u8);
69        } else {
70            out.extend_from_slice(c.encode_utf8(&mut buf).as_bytes());
71        }
72    }
73    Cow::Owned(out)
74}
75
76/// Writes `msg` to stderr as C's bytes, as cgraph's `default_usererrf`
77/// (`lib/cgraph/agerror.c`) does: up to the first NUL, with control characters other than
78/// white space written as `\ooo`.
79pub fn eprint_bytes(msg: &str) {
80    let bytes = encode(msg);
81    let bytes = bytes.split(|&b| b == 0).next().unwrap_or_default();
82    let mut out = Vec::with_capacity(bytes.len());
83    for &b in bytes {
84        // gv_iscntrl && !gv_isspace
85        if (b <= 31 || b == 127) && !matches!(b, b'\t' | b'\n' | 0x0b | 0x0c | b'\r') {
86            out.extend_from_slice(format!("\\{b:03o}").as_bytes());
87        } else {
88            out.push(b);
89        }
90    }
91    let _ = std::io::stderr().write_all(&out);
92}
93
94/// `GD_charset(g)`: the input character set of a graph (`CHAR_UTF8`, `CHAR_LATIN1`,
95/// `CHAR_BIG5` in `lib/common/const.h`).
96#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
97pub enum Charset {
98    /// `CHAR_UTF8` (the default, and the fallback for unknown names).
99    #[default]
100    Utf8,
101    /// `CHAR_LATIN1`.
102    Latin1,
103    /// `CHAR_BIG5`: labels are converted like UTF-8, but `make_simple_label` keeps the
104    /// byte after a Big5 lead byte (`0xA1..=0xFE`) with it, and HTML labels fail
105    /// (expat has no `BIG-5` decoder).
106    Big5,
107}
108
109impl Charset {
110    /// `findCharset` from `lib/common/input.c`. The second value is `false` when the name
111    /// is not recognised (C warns and assumes UTF-8).
112    pub fn parse(charset: Option<&str>) -> (Self, bool) {
113        let p = charset.unwrap_or("utf-8");
114        let is = |names: &[&str]| names.iter().any(|n| n.eq_ignore_ascii_case(p));
115        if is(&[
116            "latin-1",
117            "latin1",
118            "l1",
119            "ISO-8859-1",
120            "ISO_8859-1",
121            "ISO8859-1",
122            "ISO-IR-100",
123        ]) {
124            (Self::Latin1, true)
125        } else if is(&["big-5", "big5"]) {
126            (Self::Big5, true)
127        } else if is(&["utf-8", "utf8"]) {
128            (Self::Utf8, true)
129        } else {
130            (Self::Utf8, false)
131        }
132    }
133
134    /// [`Self::parse`] without the validity flag.
135    pub fn from_attr(charset: Option<&str>) -> Self {
136        Self::parse(charset).0
137    }
138}
139
140thread_local! {
141    static CURRENT: Cell<Charset> = const { Cell::new(Charset::Utf8) };
142    static GRAPH: RefCell<String> = const { RefCell::new(String::new()) };
143    static WARNED: Cell<bool> = const { Cell::new(false) };
144}
145
146/// `graph_init` for charset purposes: records the root graph's charset and name for the
147/// label conversions that follow, clears `htmlEntityUTF8`'s once-per-graph warning flag
148/// and warns about an unsupported charset name.
149pub fn begin_layout(graph_name: &str, charset: Option<&str>) {
150    let (cs, ok) = Charset::parse(charset);
151    if !ok {
152        eprint_bytes(&format!(
153            "Warning: Unsupported charset \"{}\" - assuming utf-8\n",
154            charset.unwrap_or("")
155        ));
156    }
157    CURRENT.with(|c| c.set(cs));
158    WARNED.with(|w| w.set(false));
159    GRAPH.with(|g| {
160        let mut g = g.borrow_mut();
161        g.clear();
162        g.push_str(graph_name);
163    });
164}
165
166/// Sets the charset of the graph being rendered (no warnings; the layout already gave them).
167pub fn begin_render(graph_name: &str, charset: Option<&str>) {
168    let cs = Charset::from_attr(charset);
169    CURRENT.with(|c| c.set(cs));
170    GRAPH.with(|g| {
171        let mut g = g.borrow_mut();
172        if *g != graph_name {
173            g.clear();
174            g.push_str(graph_name);
175            WARNED.with(|w| w.set(false));
176        }
177    });
178}
179
180/// Charset of the graph being laid out or rendered on this thread.
181pub fn current() -> Charset {
182    CURRENT.with(Cell::get)
183}
184
185/// `htmlEntityUTF8`'s warning: printed at most once per graph.
186pub(crate) fn warn_invalid_utf8(msg: impl FnOnce(&str) -> String) {
187    if WARNED.with(|w| w.replace(true)) {
188        return;
189    }
190    let text = GRAPH.with(|g| msg(&g.borrow()));
191    eprint_bytes(&text);
192}
193
194#[cfg(test)]
195mod tests {
196    use super::*;
197
198    #[test]
199    fn round_trip() {
200        let bytes = b"caf\xe9 \xc3\xa9 \xe4\xbcz \xff".to_vec();
201        let s = decode(bytes.clone());
202        assert!(s.contains('\u{e9}'));
203        assert!(has_raw(&s));
204        assert_eq!(&*encode(&s), &bytes[..]);
205        assert_eq!(decode(b"plain".to_vec()), "plain");
206        assert!(matches!(encode("plain \u{10ffff}x"), Cow::Owned(_)));
207    }
208
209    #[test]
210    fn find_charset() {
211        assert_eq!(Charset::parse(Some("ISO-8859-1")), (Charset::Latin1, true));
212        assert_eq!(Charset::parse(Some("BIG5")), (Charset::Big5, true));
213        assert_eq!(Charset::parse(None), (Charset::Utf8, true));
214        assert_eq!(Charset::parse(Some("koi8-r")), (Charset::Utf8, false));
215    }
216}
graphviz_pure_core/charset.rs

graphviz_pure_core/
charset.rs
