From e989fc2b3d2dadc8a6a2d2430c389f5fb1f94812 Mon Sep 17 00:00:00 2001 From: xmaks82 Date: Mon, 12 Oct 2026 18:36:15 +0300 Subject: [PATCH] Open RTF in its code page, and Word/RTF files whose extension is wrong (#896) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit RTF import read every 8-bit byte as Windows-1252, so Cyrillic (and any other non-Western) RTF came out as mojibake. Bytes now decode in the current font's code page (\fcharset, \cpg), else the default font's, else the document's (\ansicpg), for the single-byte Windows code pages 874 and 1250-1258. A .doc, .docx or .rtf whose content is another of these formats now opens by its signature, as Word does: RTF saved as .doc (hh.ru résumés and accounting exports are commonly saved that way), a .docx renamed .doc, a Word 97-2003 file renamed .docx. Co-authored-by: Магда М.Г. Co-authored-by: Brandon Thomas --- crates/doc/src/encoding.rs | 177 ++++++++++++++++++++++++++++++++++++- crates/engine/src/io.rs | 27 +++++- crates/engine/src/tests.rs | 25 ++++++ crates/formats/src/rtf.rs | 72 +++++++++++++-- docs/file-format-parity.md | 5 +- 5 files changed, 295 insertions(+), 11 deletions(-) diff --git a/crates/doc/src/encoding.rs b/crates/doc/src/encoding.rs index 29dc569..818ef39 100644 --- a/crates/doc/src/encoding.rs +++ b/crates/doc/src/encoding.rs @@ -1,4 +1,6 @@ -//! Windows-1252 (the byte encoding legacy Word text pieces and RTF bytes use). +//! Windows-1252 (the byte encoding legacy Word text pieces use) and the other single-byte Windows +//! code pages RTF files name with `\ansicpg` or a font's `\fcharset` (Cyrillic 1251, Central +//! European 1250, Greek, Turkish, Hebrew, Arabic, Baltic, Vietnamese, Thai). /// Windows-1252 code points for 0x80..=0x9F. const CP1252: [u16; 32] = [ @@ -14,3 +16,176 @@ pub fn cp1252(b: u8) -> char { b as char } } + +/// Windows-874 code points for 0x80..=0xFF (U+FFFD where the byte is unassigned). +const CP874: [u16; 128] = [ + 0x20AC, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0x2026, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0x2018, + 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0x00A0, 0x0E01, 0x0E02, 0x0E03, + 0x0E04, 0x0E05, 0x0E06, 0x0E07, 0x0E08, 0x0E09, 0x0E0A, 0x0E0B, 0x0E0C, 0x0E0D, 0x0E0E, 0x0E0F, 0x0E10, 0x0E11, 0x0E12, 0x0E13, 0x0E14, 0x0E15, + 0x0E16, 0x0E17, 0x0E18, 0x0E19, 0x0E1A, 0x0E1B, 0x0E1C, 0x0E1D, 0x0E1E, 0x0E1F, 0x0E20, 0x0E21, 0x0E22, 0x0E23, 0x0E24, 0x0E25, 0x0E26, 0x0E27, + 0x0E28, 0x0E29, 0x0E2A, 0x0E2B, 0x0E2C, 0x0E2D, 0x0E2E, 0x0E2F, 0x0E30, 0x0E31, 0x0E32, 0x0E33, 0x0E34, 0x0E35, 0x0E36, 0x0E37, 0x0E38, 0x0E39, + 0x0E3A, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0x0E3F, 0x0E40, 0x0E41, 0x0E42, 0x0E43, 0x0E44, 0x0E45, 0x0E46, 0x0E47, 0x0E48, 0x0E49, 0x0E4A, 0x0E4B, + 0x0E4C, 0x0E4D, 0x0E4E, 0x0E4F, 0x0E50, 0x0E51, 0x0E52, 0x0E53, 0x0E54, 0x0E55, 0x0E56, 0x0E57, 0x0E58, 0x0E59, 0x0E5A, 0x0E5B, 0xFFFD, 0xFFFD, + 0xFFFD, 0xFFFD, +]; + +/// Windows-1250 code points for 0x80..=0xFF (U+FFFD where the byte is unassigned). +const CP1250: [u16; 128] = [ + 0x20AC, 0xFFFD, 0x201A, 0xFFFD, 0x201E, 0x2026, 0x2020, 0x2021, 0xFFFD, 0x2030, 0x0160, 0x2039, 0x015A, 0x0164, 0x017D, 0x0179, 0xFFFD, 0x2018, + 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014, 0xFFFD, 0x2122, 0x0161, 0x203A, 0x015B, 0x0165, 0x017E, 0x017A, 0x00A0, 0x02C7, 0x02D8, 0x0141, + 0x00A4, 0x0104, 0x00A6, 0x00A7, 0x00A8, 0x00A9, 0x015E, 0x00AB, 0x00AC, 0x00AD, 0x00AE, 0x017B, 0x00B0, 0x00B1, 0x02DB, 0x0142, 0x00B4, 0x00B5, + 0x00B6, 0x00B7, 0x00B8, 0x0105, 0x015F, 0x00BB, 0x013D, 0x02DD, 0x013E, 0x017C, 0x0154, 0x00C1, 0x00C2, 0x0102, 0x00C4, 0x0139, 0x0106, 0x00C7, + 0x010C, 0x00C9, 0x0118, 0x00CB, 0x011A, 0x00CD, 0x00CE, 0x010E, 0x0110, 0x0143, 0x0147, 0x00D3, 0x00D4, 0x0150, 0x00D6, 0x00D7, 0x0158, 0x016E, + 0x00DA, 0x0170, 0x00DC, 0x00DD, 0x0162, 0x00DF, 0x0155, 0x00E1, 0x00E2, 0x0103, 0x00E4, 0x013A, 0x0107, 0x00E7, 0x010D, 0x00E9, 0x0119, 0x00EB, + 0x011B, 0x00ED, 0x00EE, 0x010F, 0x0111, 0x0144, 0x0148, 0x00F3, 0x00F4, 0x0151, 0x00F6, 0x00F7, 0x0159, 0x016F, 0x00FA, 0x0171, 0x00FC, 0x00FD, + 0x0163, 0x02D9, +]; + +/// Windows-1251 code points for 0x80..=0xFF (U+FFFD where the byte is unassigned). +const CP1251: [u16; 128] = [ + 0x0402, 0x0403, 0x201A, 0x0453, 0x201E, 0x2026, 0x2020, 0x2021, 0x20AC, 0x2030, 0x0409, 0x2039, 0x040A, 0x040C, 0x040B, 0x040F, 0x0452, 0x2018, + 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014, 0xFFFD, 0x2122, 0x0459, 0x203A, 0x045A, 0x045C, 0x045B, 0x045F, 0x00A0, 0x040E, 0x045E, 0x0408, + 0x00A4, 0x0490, 0x00A6, 0x00A7, 0x0401, 0x00A9, 0x0404, 0x00AB, 0x00AC, 0x00AD, 0x00AE, 0x0407, 0x00B0, 0x00B1, 0x0406, 0x0456, 0x0491, 0x00B5, + 0x00B6, 0x00B7, 0x0451, 0x2116, 0x0454, 0x00BB, 0x0458, 0x0405, 0x0455, 0x0457, 0x0410, 0x0411, 0x0412, 0x0413, 0x0414, 0x0415, 0x0416, 0x0417, + 0x0418, 0x0419, 0x041A, 0x041B, 0x041C, 0x041D, 0x041E, 0x041F, 0x0420, 0x0421, 0x0422, 0x0423, 0x0424, 0x0425, 0x0426, 0x0427, 0x0428, 0x0429, + 0x042A, 0x042B, 0x042C, 0x042D, 0x042E, 0x042F, 0x0430, 0x0431, 0x0432, 0x0433, 0x0434, 0x0435, 0x0436, 0x0437, 0x0438, 0x0439, 0x043A, 0x043B, + 0x043C, 0x043D, 0x043E, 0x043F, 0x0440, 0x0441, 0x0442, 0x0443, 0x0444, 0x0445, 0x0446, 0x0447, 0x0448, 0x0449, 0x044A, 0x044B, 0x044C, 0x044D, + 0x044E, 0x044F, +]; + +/// Windows-1253 code points for 0x80..=0xFF (U+FFFD where the byte is unassigned). +const CP1253: [u16; 128] = [ + 0x20AC, 0xFFFD, 0x201A, 0x0192, 0x201E, 0x2026, 0x2020, 0x2021, 0xFFFD, 0x2030, 0xFFFD, 0x2039, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0x2018, + 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014, 0xFFFD, 0x2122, 0xFFFD, 0x203A, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0x00A0, 0x0385, 0x0386, 0x00A3, + 0x00A4, 0x00A5, 0x00A6, 0x00A7, 0x00A8, 0x00A9, 0xFFFD, 0x00AB, 0x00AC, 0x00AD, 0x00AE, 0x2015, 0x00B0, 0x00B1, 0x00B2, 0x00B3, 0x0384, 0x00B5, + 0x00B6, 0x00B7, 0x0388, 0x0389, 0x038A, 0x00BB, 0x038C, 0x00BD, 0x038E, 0x038F, 0x0390, 0x0391, 0x0392, 0x0393, 0x0394, 0x0395, 0x0396, 0x0397, + 0x0398, 0x0399, 0x039A, 0x039B, 0x039C, 0x039D, 0x039E, 0x039F, 0x03A0, 0x03A1, 0xFFFD, 0x03A3, 0x03A4, 0x03A5, 0x03A6, 0x03A7, 0x03A8, 0x03A9, + 0x03AA, 0x03AB, 0x03AC, 0x03AD, 0x03AE, 0x03AF, 0x03B0, 0x03B1, 0x03B2, 0x03B3, 0x03B4, 0x03B5, 0x03B6, 0x03B7, 0x03B8, 0x03B9, 0x03BA, 0x03BB, + 0x03BC, 0x03BD, 0x03BE, 0x03BF, 0x03C0, 0x03C1, 0x03C2, 0x03C3, 0x03C4, 0x03C5, 0x03C6, 0x03C7, 0x03C8, 0x03C9, 0x03CA, 0x03CB, 0x03CC, 0x03CD, + 0x03CE, 0xFFFD, +]; + +/// Windows-1254 code points for 0x80..=0xFF (U+FFFD where the byte is unassigned). +const CP1254: [u16; 128] = [ + 0x20AC, 0xFFFD, 0x201A, 0x0192, 0x201E, 0x2026, 0x2020, 0x2021, 0x02C6, 0x2030, 0x0160, 0x2039, 0x0152, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0x2018, + 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014, 0x02DC, 0x2122, 0x0161, 0x203A, 0x0153, 0xFFFD, 0xFFFD, 0x0178, 0x00A0, 0x00A1, 0x00A2, 0x00A3, + 0x00A4, 0x00A5, 0x00A6, 0x00A7, 0x00A8, 0x00A9, 0x00AA, 0x00AB, 0x00AC, 0x00AD, 0x00AE, 0x00AF, 0x00B0, 0x00B1, 0x00B2, 0x00B3, 0x00B4, 0x00B5, + 0x00B6, 0x00B7, 0x00B8, 0x00B9, 0x00BA, 0x00BB, 0x00BC, 0x00BD, 0x00BE, 0x00BF, 0x00C0, 0x00C1, 0x00C2, 0x00C3, 0x00C4, 0x00C5, 0x00C6, 0x00C7, + 0x00C8, 0x00C9, 0x00CA, 0x00CB, 0x00CC, 0x00CD, 0x00CE, 0x00CF, 0x011E, 0x00D1, 0x00D2, 0x00D3, 0x00D4, 0x00D5, 0x00D6, 0x00D7, 0x00D8, 0x00D9, + 0x00DA, 0x00DB, 0x00DC, 0x0130, 0x015E, 0x00DF, 0x00E0, 0x00E1, 0x00E2, 0x00E3, 0x00E4, 0x00E5, 0x00E6, 0x00E7, 0x00E8, 0x00E9, 0x00EA, 0x00EB, + 0x00EC, 0x00ED, 0x00EE, 0x00EF, 0x011F, 0x00F1, 0x00F2, 0x00F3, 0x00F4, 0x00F5, 0x00F6, 0x00F7, 0x00F8, 0x00F9, 0x00FA, 0x00FB, 0x00FC, 0x0131, + 0x015F, 0x00FF, +]; + +/// Windows-1255 code points for 0x80..=0xFF (U+FFFD where the byte is unassigned). +const CP1255: [u16; 128] = [ + 0x20AC, 0xFFFD, 0x201A, 0x0192, 0x201E, 0x2026, 0x2020, 0x2021, 0x02C6, 0x2030, 0xFFFD, 0x2039, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0x2018, + 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014, 0x02DC, 0x2122, 0xFFFD, 0x203A, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0x00A0, 0x00A1, 0x00A2, 0x00A3, + 0x20AA, 0x00A5, 0x00A6, 0x00A7, 0x00A8, 0x00A9, 0x00D7, 0x00AB, 0x00AC, 0x00AD, 0x00AE, 0x00AF, 0x00B0, 0x00B1, 0x00B2, 0x00B3, 0x00B4, 0x00B5, + 0x00B6, 0x00B7, 0x00B8, 0x00B9, 0x00F7, 0x00BB, 0x00BC, 0x00BD, 0x00BE, 0x00BF, 0x05B0, 0x05B1, 0x05B2, 0x05B3, 0x05B4, 0x05B5, 0x05B6, 0x05B7, + 0x05B8, 0x05B9, 0xFFFD, 0x05BB, 0x05BC, 0x05BD, 0x05BE, 0x05BF, 0x05C0, 0x05C1, 0x05C2, 0x05C3, 0x05F0, 0x05F1, 0x05F2, 0x05F3, 0x05F4, 0xFFFD, + 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0x05D0, 0x05D1, 0x05D2, 0x05D3, 0x05D4, 0x05D5, 0x05D6, 0x05D7, 0x05D8, 0x05D9, 0x05DA, 0x05DB, + 0x05DC, 0x05DD, 0x05DE, 0x05DF, 0x05E0, 0x05E1, 0x05E2, 0x05E3, 0x05E4, 0x05E5, 0x05E6, 0x05E7, 0x05E8, 0x05E9, 0x05EA, 0xFFFD, 0xFFFD, 0x200E, + 0x200F, 0xFFFD, +]; + +/// Windows-1256 code points for 0x80..=0xFF (U+FFFD where the byte is unassigned). +const CP1256: [u16; 128] = [ + 0x20AC, 0x067E, 0x201A, 0x0192, 0x201E, 0x2026, 0x2020, 0x2021, 0x02C6, 0x2030, 0x0679, 0x2039, 0x0152, 0x0686, 0x0698, 0x0688, 0x06AF, 0x2018, + 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014, 0x06A9, 0x2122, 0x0691, 0x203A, 0x0153, 0x200C, 0x200D, 0x06BA, 0x00A0, 0x060C, 0x00A2, 0x00A3, + 0x00A4, 0x00A5, 0x00A6, 0x00A7, 0x00A8, 0x00A9, 0x06BE, 0x00AB, 0x00AC, 0x00AD, 0x00AE, 0x00AF, 0x00B0, 0x00B1, 0x00B2, 0x00B3, 0x00B4, 0x00B5, + 0x00B6, 0x00B7, 0x00B8, 0x00B9, 0x061B, 0x00BB, 0x00BC, 0x00BD, 0x00BE, 0x061F, 0x06C1, 0x0621, 0x0622, 0x0623, 0x0624, 0x0625, 0x0626, 0x0627, + 0x0628, 0x0629, 0x062A, 0x062B, 0x062C, 0x062D, 0x062E, 0x062F, 0x0630, 0x0631, 0x0632, 0x0633, 0x0634, 0x0635, 0x0636, 0x00D7, 0x0637, 0x0638, + 0x0639, 0x063A, 0x0640, 0x0641, 0x0642, 0x0643, 0x00E0, 0x0644, 0x00E2, 0x0645, 0x0646, 0x0647, 0x0648, 0x00E7, 0x00E8, 0x00E9, 0x00EA, 0x00EB, + 0x0649, 0x064A, 0x00EE, 0x00EF, 0x064B, 0x064C, 0x064D, 0x064E, 0x00F4, 0x064F, 0x0650, 0x00F7, 0x0651, 0x00F9, 0x0652, 0x00FB, 0x00FC, 0x200E, + 0x200F, 0x06D2, +]; + +/// Windows-1257 code points for 0x80..=0xFF (U+FFFD where the byte is unassigned). +const CP1257: [u16; 128] = [ + 0x20AC, 0xFFFD, 0x201A, 0xFFFD, 0x201E, 0x2026, 0x2020, 0x2021, 0xFFFD, 0x2030, 0xFFFD, 0x2039, 0xFFFD, 0x00A8, 0x02C7, 0x00B8, 0xFFFD, 0x2018, + 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014, 0xFFFD, 0x2122, 0xFFFD, 0x203A, 0xFFFD, 0x00AF, 0x02DB, 0xFFFD, 0x00A0, 0xFFFD, 0x00A2, 0x00A3, + 0x00A4, 0xFFFD, 0x00A6, 0x00A7, 0x00D8, 0x00A9, 0x0156, 0x00AB, 0x00AC, 0x00AD, 0x00AE, 0x00C6, 0x00B0, 0x00B1, 0x00B2, 0x00B3, 0x00B4, 0x00B5, + 0x00B6, 0x00B7, 0x00F8, 0x00B9, 0x0157, 0x00BB, 0x00BC, 0x00BD, 0x00BE, 0x00E6, 0x0104, 0x012E, 0x0100, 0x0106, 0x00C4, 0x00C5, 0x0118, 0x0112, + 0x010C, 0x00C9, 0x0179, 0x0116, 0x0122, 0x0136, 0x012A, 0x013B, 0x0160, 0x0143, 0x0145, 0x00D3, 0x014C, 0x00D5, 0x00D6, 0x00D7, 0x0172, 0x0141, + 0x015A, 0x016A, 0x00DC, 0x017B, 0x017D, 0x00DF, 0x0105, 0x012F, 0x0101, 0x0107, 0x00E4, 0x00E5, 0x0119, 0x0113, 0x010D, 0x00E9, 0x017A, 0x0117, + 0x0123, 0x0137, 0x012B, 0x013C, 0x0161, 0x0144, 0x0146, 0x00F3, 0x014D, 0x00F5, 0x00F6, 0x00F7, 0x0173, 0x0142, 0x015B, 0x016B, 0x00FC, 0x017C, + 0x017E, 0x02D9, +]; + +/// Windows-1258 code points for 0x80..=0xFF (U+FFFD where the byte is unassigned). +const CP1258: [u16; 128] = [ + 0x20AC, 0xFFFD, 0x201A, 0x0192, 0x201E, 0x2026, 0x2020, 0x2021, 0x02C6, 0x2030, 0xFFFD, 0x2039, 0x0152, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0x2018, + 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014, 0x02DC, 0x2122, 0xFFFD, 0x203A, 0x0153, 0xFFFD, 0xFFFD, 0x0178, 0x00A0, 0x00A1, 0x00A2, 0x00A3, + 0x00A4, 0x00A5, 0x00A6, 0x00A7, 0x00A8, 0x00A9, 0x00AA, 0x00AB, 0x00AC, 0x00AD, 0x00AE, 0x00AF, 0x00B0, 0x00B1, 0x00B2, 0x00B3, 0x00B4, 0x00B5, + 0x00B6, 0x00B7, 0x00B8, 0x00B9, 0x00BA, 0x00BB, 0x00BC, 0x00BD, 0x00BE, 0x00BF, 0x00C0, 0x00C1, 0x00C2, 0x0102, 0x00C4, 0x00C5, 0x00C6, 0x00C7, + 0x00C8, 0x00C9, 0x00CA, 0x00CB, 0x0300, 0x00CD, 0x00CE, 0x00CF, 0x0110, 0x00D1, 0x0309, 0x00D3, 0x00D4, 0x01A0, 0x00D6, 0x00D7, 0x00D8, 0x00D9, + 0x00DA, 0x00DB, 0x00DC, 0x01AF, 0x0303, 0x00DF, 0x00E0, 0x00E1, 0x00E2, 0x0103, 0x00E4, 0x00E5, 0x00E6, 0x00E7, 0x00E8, 0x00E9, 0x00EA, 0x00EB, + 0x0301, 0x00ED, 0x00EE, 0x00EF, 0x0111, 0x00F1, 0x0323, 0x00F3, 0x00F4, 0x01A1, 0x00F6, 0x00F7, 0x00F8, 0x00F9, 0x00FA, 0x00FB, 0x00FC, 0x01B0, + 0x20AB, 0x00FF, +]; + +/// The single-byte Windows code pages [`windows_byte`] decodes (1252 included). +pub const SINGLE_BYTE_CODE_PAGES: [u16; 10] = [874, 1250, 1251, 1252, 1253, 1254, 1255, 1256, 1257, 1258]; + +/// One byte of a single-byte Windows code page as a char; a code page outside +/// [`SINGLE_BYTE_CODE_PAGES`] reads as Windows-1252. +pub fn windows_byte(code_page: u16, b: u8) -> char { + let table: &[u16; 128] = match code_page { + 874 => &CP874, + 1250 => &CP1250, + 1251 => &CP1251, + 1253 => &CP1253, + 1254 => &CP1254, + 1255 => &CP1255, + 1256 => &CP1256, + 1257 => &CP1257, + 1258 => &CP1258, + _ => return cp1252(b), + }; + if b < 0x80 { + return b as char; + } + table.get(usize::from(b - 0x80)).and_then(|u| char::from_u32(u32::from(*u))).unwrap_or('\u{FFFD}') +} + +/// The code page of an RTF or Word font charset (`\fcharset`): 204 Russian → 1251, 238 Eastern +/// European → 1250… `None` for the ANSI and default charsets (the document's code page applies), +/// symbol fonts and the double-byte East Asian charsets. +pub fn charset_code_page(charset: i32) -> Option { + Some(match charset { + 161 => 1253, + 162 => 1254, + 163 => 1258, + 177 => 1255, + 178 => 1256, + 186 => 1257, + 204 => 1251, + 222 => 874, + 238 => 1250, + _ => return None, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn single_byte_code_pages() { + let decode = |cp: u16, bytes: &[u8]| bytes.iter().map(|&b| windows_byte(cp, b)).collect::(); + assert_eq!(decode(1251, &[0xD1, 0xF7, 0xE5, 0xF2, 0x20, 0xB9]), "Счет №"); + assert_eq!(decode(1251, &[0xA8, 0xB8]), "Ёё"); + assert_eq!(decode(1250, &[0xA3, 0xF3, 0x9F]), "Łóź"); + assert_eq!(decode(1253, &[0xC1, 0xE2]), "Αβ"); + assert_eq!(decode(1252, &[0x80, 0xE9]), "€é"); + assert_eq!(decode(9999, &[0xE9]), "é"); + assert_eq!(charset_code_page(204), Some(1251)); + assert_eq!(charset_code_page(0), None); + for cp in SINGLE_BYTE_CODE_PAGES { + for b in 0..=255u8 { + let _ = windows_byte(cp, b); + } + } + } +} diff --git a/crates/engine/src/io.rs b/crates/engine/src/io.rs index cdc1d63..6e2e938 100644 --- a/crates/engine/src/io.rs +++ b/crates/engine/src/io.rs @@ -110,7 +110,7 @@ pub fn open_bytes_with(name: &str, bytes: &[u8], password: Option<&str>) -> Resu doc.ensure_nonempty(); return Ok((doc, true)); } - if ext.is_empty() { + if ext.is_empty() || signature_contradicts(&ext, bytes) { return open_by_content(name, &ext, bytes, password); } let mut doc = match ext.as_str() { @@ -163,6 +163,31 @@ pub(crate) fn fit_pictures(doc: &mut Document) { } } +/// Does the file hold a different document format than its Word or RTF extension says? An RTF +/// file saved as `.doc` (as accounting systems and report generators often save their +/// documents), a `.docx` package renamed `.doc` or a Word 97-2003 file renamed `.docx`: Word opens +/// each by what it holds, so they go through [`open_by_content`] instead of failing. +fn signature_contradicts(ext: &str, bytes: &[u8]) -> bool { + const ZIP: &[u8] = b"PK\x03\x04"; + const OLE: &[u8] = &[0xD0, 0xCF, 0x11, 0xE0, 0xA1, 0xB1, 0x1A, 0xE1]; + let held = if bytes.starts_with(ZIP) { + "zip" + } else if bytes.starts_with(OLE) { + "ole" + } else if bytes.starts_with(b"{\\rtf") { + "rtf" + } else { + return false; + }; + let expected = match ext { + "docx" | "docm" | "dotx" | "dotm" => "zip", + "doc" | "dot" => "ole", + "rtf" => "rtf", + _ => return false, + }; + held != expected +} + /// A file whose extension names no format WordCraft knows (`.m3u`, `.log`, `.ini`, none…), /// opened by what it holds, as Word does: a Word document or RTF file by its signature, any /// other text through the plain-text import with encoding detection (#569). Anything else is diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index a722ce9..785c11d 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -3732,6 +3732,31 @@ fn colour_commands_take_any_custom_hex() { assert_eq!(s.doc.settings.page_color, wordcraft_doc::Rgb::parse("123ABC")); } +/// A document whose content is another format than its extension says opens by what it holds, +/// as in Word: RTF saved as `.doc` (accounting systems and report generators save their documents +/// that way), a `.docx` package renamed `.doc`, and RTF renamed `.docx`. +#[test] +fn documents_open_by_content_when_the_extension_is_wrong() { + let text = |doc: &wordcraft_doc::Document| doc.plain_text(StoryRef::Body); + let rtf = b"{\\rtf1\\ansi\\ansicpg1251\\deff0{\\fonttbl{\\f0 Times New Roman;}}\\f0 \\'d1\\'f7\\'e5\\'f2 \\'b9 12\\par Invoice line\\par}"; + for name in ["Счет 12.doc", "report.DOC", "letter.dot", "renamed.docx"] { + let doc = crate::io::open_bytes(name, rtf).unwrap_or_else(|e| panic!("{name}: {e}")); + let t = text(&doc); + assert!(t.contains("Счет № 12") && t.contains("Invoice line"), "{name}: {t:?}"); + } + // A Word package renamed `.doc` or `.rtf`. + let mut s = Session::new(wordcraft_doc::Document::new()); + run(&mut s, "text.insert", json!({"text": "Packaged text"})); + let docx = crate::io::save_bytes("a.docx", &s.doc).unwrap(); + for name in ["package.doc", "package.rtf"] { + let doc = crate::io::open_bytes(name, &docx).unwrap_or_else(|e| panic!("{name}: {e}")); + assert!(text(&doc).contains("Packaged text"), "{name}"); + } + // Matching content still goes the usual way, and garbage stays an error. + assert!(text(&crate::io::open_bytes("plain.rtf", rtf).unwrap()).contains("Invoice line")); + assert!(crate::io::open_bytes("broken.doc", b"\x00\x01not a document").is_err()); +} + /// #569: a playlist (.m3u / .m3u8) — or any file with an extension WordCraft doesn't know — /// opens as plain text with its encoding detected, like Word's text import; a binary file is a /// clean error, never garbage or a panic. diff --git a/crates/formats/src/rtf.rs b/crates/formats/src/rtf.rs index 0e57fa5..8e7f4bb 100644 --- a/crates/formats/src/rtf.rs +++ b/crates/formats/src/rtf.rs @@ -9,7 +9,9 @@ //! //! Import: a tolerant reader for groups and destinations (unknown `\*` destinations, headers, //! footers, notes and objects are skipped), `\par \line \tab \page`, `\uN` with `\ucN` -//! fallback skipping, `\'hh` (Windows-1252), the formatting words above, stylesheet-based +//! fallback skipping, `\'hh` and 8-bit text in the document's code page (`\ansicpg`) or the +//! current font's (`\fcharset`, `\cpg`: Cyrillic, Central European, Greek…; Windows-1252 when +//! none is named), the formatting words above, stylesheet-based //! headings and `\outlinelevel`, lists through `\ls`/`\ilvl`, fields, bookmarks, pictures and //! tables. Group nesting and picture sizes are capped. @@ -18,7 +20,6 @@ use std::collections::HashMap; use wordcraft_doc::{Align, Document, Rgb}; use crate::model::{self, Cell, FBlock, FTable, Flow, Fmt, Inline, Kind, ListInfo, Meta, Para, make_img}; -use crate::txt::cp1252; /// Deepest group nesting we track; deeper groups share the innermost state. const MAX_GROUPS: usize = 512; @@ -496,6 +497,8 @@ struct St { outline: Option, pagebb: bool, field: Option, + /// The code page of the current font's charset (`\fcharset`), when it names one. + cp: Option, } impl Default for St { @@ -513,6 +516,7 @@ impl Default for St { outline: None, pagebb: false, field: None, + cp: None, } } } @@ -533,6 +537,13 @@ struct Reader { fonts: HashMap, font_cur: i32, font_name: String, + /// Code pages of the fonts whose charset names one (`\fcharset204` → 1251). + font_cps: HashMap, + font_cp_cur: Option, + /// The document's code page (`\ansicpg`); 0 until it names one (Windows-1252 then). + ansicpg: u16, + /// The default font (`\deff`), whose charset applies before any `\f`. + deff: i32, colors: Vec>, color_cur: (u8, u8, u8, bool), styles: HashMap, @@ -637,9 +648,19 @@ impl Reader { let name = std::mem::take(&mut self.font_name).trim().trim_end_matches(';').trim().to_string(); if !name.is_empty() && self.fonts.len() < 4096 { self.fonts.insert(self.font_cur, name); + if let Some(cp) = self.font_cp_cur.take() { + self.font_cps.insert(self.font_cur, cp); + } } } + /// A byte of 8-bit text (`\'hh` or raw) as a char, in the current font's code page, else the + /// default font's, else the document's. + fn byte_char(&self, b: u8) -> char { + let cp = self.st.cp.or_else(|| self.font_cps.get(&self.deff).copied()).unwrap_or(if self.ansicpg == 0 { 1252 } else { self.ansicpg }); + wordcraft_doc::encoding::windows_byte(cp, b) + } + fn commit_style(&mut self) { let name = std::mem::take(&mut self.style_name).trim().trim_end_matches(';').trim().to_string(); if self.styles.len() < 4096 { @@ -962,9 +983,16 @@ impl Reader { return; } Dest::FontTbl => { - if w == "f" { - self.font_cur = pv; - self.font_name.clear(); + match w { + "f" => { + self.font_cur = pv; + self.font_name.clear(); + self.font_cp_cur = None; + } + "fcharset" => self.font_cp_cur = wordcraft_doc::encoding::charset_code_page(pv), + // A font's explicit code page (`\cpg1251`) wins over its charset. + "cpg" if pv > 0 && wordcraft_doc::encoding::SINGLE_BYTE_CODE_PAGES.contains(&(pv as u16)) => self.font_cp_cur = Some(pv as u16), + _ => {} } return; } @@ -1115,7 +1143,14 @@ impl Reader { self.st.fmt.sub = false; } "fs" => self.st.fmt.size = if pv > 0 { Some((pv as f32 / 2.0).clamp(1.0, 1638.0)) } else { None }, + "ansicpg" => { + if pv > 0 && wordcraft_doc::encoding::SINGLE_BYTE_CODE_PAGES.contains(&(pv as u16)) { + self.ansicpg = pv as u16; + } + } + "deff" => self.deff = pv, "f" => { + self.st.cp = self.font_cps.get(&pv).copied(); let name = self.fonts.get(&pv).cloned(); match name { Some(n) if model::is_mono(&n) => { @@ -1300,7 +1335,8 @@ pub fn parse(bytes: &[u8]) -> Result { if r.st.dest == Dest::Pict { r.text(&format!("{v:02x}")); } else { - r.ch(cp1252(v)); + let c = r.byte_char(v); + r.ch(c); } i += 4; } @@ -1350,7 +1386,8 @@ pub fn parse(bytes: &[u8]) -> Result { } } else { for b in chunk { - r.ch(cp1252(*b)); + let c = r.byte_char(*b); + r.ch(c); } } i = j; @@ -1420,6 +1457,27 @@ mod tests { assert!(matches!(&l.inlines[0], Inline::Text(_, f) if f.link.as_deref() == Some("http://a.b"))); } + #[test] + fn code_pages_from_ansicpg_and_font_charsets() { + // The document's code page: Russian RTF as accounting systems write it. + let f = parse(br"{\rtf1\ansi\ansicpg1251\deff0{\fonttbl{\f0 Times New Roman;}}\f0 \'cf\'f0\'e8\'e2\'e5\'f2, \'b8\'eb\'ea\'e0 \'b9 5\par}") + .unwrap(); + assert_eq!(texts(&f), ["Normal:Привет, ёлка № 5"]); + // Font charsets switch inside a Windows-1252 document: Western, Cyrillic, Central + // European, then back to the first font. + let f = parse( + br"{\rtf1\ansi\ansicpg1252\deff0{\fonttbl{\f0 Arial;}{\f1\fcharset204 Arial CYR;}{\f2\fcharset238 Arial CE;}}\f0 caf\'e9 {\f1 \'c4\'e0} \f2 \'a3\'f3d\'9f \f0 na\'efve\par}", + ) + .unwrap(); + assert_eq!(texts(&f), ["Normal:café Да Łódź naïve"]); + // The default font's charset applies before any \f; an explicit \cpg wins over \fcharset. + let f = parse(br"{\rtf1\ansi\deff1{\fonttbl{\f1\fcharset204 Times;}{\f2\fcharset0\cpg1253 Greek;}}\'cc\'e8\'f0 \f2 \'c1\'e2\par}").unwrap(); + assert_eq!(texts(&f), ["Normal:Мир Αβ"]); + // Raw 8-bit text (not escaped) follows the same code page. + let f = parse(b"{\\rtf1\\ansi\\ansicpg1251 \xc4\xee\xec\\par}").unwrap(); + assert_eq!(texts(&f), ["Normal:Дом"]); + } + #[test] fn shppict_imports_one_image_and_skips_legacy_duplicate() -> Result<(), String> { let pixels = image::RgbaImage::from_pixel(1, 1, image::Rgba([40, 80, 160, 255])); diff --git a/docs/file-format-parity.md b/docs/file-format-parity.md index c5e3ad9..200f259 100644 --- a/docs/file-format-parity.md +++ b/docs/file-format-parity.md @@ -25,9 +25,9 @@ Weights are the share of a Word user's file traffic (estimate). Read/write: ✅ | `.doc` Word 97-2003 (and `.dot`) | 8% | R/W | 🟡 | ❌ | ~60% | 25–40 | `crates/docbin/src/tests.rs`, `tests/drawing.rs` (56) | Spec-based reader ([MS-DOC]): text, formatting, styles, sections, headers/footers, tables, lists, notes, fields, bookmarks; inline and floating pictures (PNG, JPEG, DIB, TIFF, EMF, WMF; Macintosh PICT as a sized placeholder) with crop, alt text, position and wrap; basic shapes, lines, groups and text boxes ([MS-ODRAW]). Freeform shapes, z-order and Word 6/95 or encrypted files are dropped or refused. No writer | | PDF export | 8% | W | — | ✅ | ~80% | 6–10 | `crates/pdf/src/tests.rs` | krilla: embedded subsets, selectable text incl. field results (#133), links, outline, tagging. Fonts it can't subset become outlines (#132). Print Markup (#589): comments in balloons beside each page (scaled to its paper) and tracked changes; HTML/Markdown/text copies end with a comment list. No PDF/A option, no "best for printing/online" choice | | PDF open (PDF Reflow, Word for Windows) | 1% | R | ❌ | — | 0% | 30–50 | — | Converting a PDF into an editable document. A sibling app (PdfCraft) may supply the parser | -| `.rtf` Rich Text Format | 3% | R/W | 🟡 | 🟡 | ~50% | 10–15 | `crates/formats/src/tests.rs` | ~1,500 lines: text, character/paragraph formatting, styles, lists, tables, pictures (`\shppict`, #154), fields, bookmarks. Thin: comments, notes, revisions, sections, RTL. Pasted from the system clipboard where the platform offers it (Windows, Wayland) when there is no HTML (#687) | +| `.rtf` Rich Text Format | 3% | R/W | 🟡 | 🟡 | ~50% | 10–15 | `crates/formats/src/tests.rs` | ~1,500 lines: text, character/paragraph formatting, styles, lists, tables, pictures (`\shppict`, #154), fields, bookmarks. 8-bit text in the document's code page (`\ansicpg`) or the current font's (`\fcharset`, `\cpg`): Cyrillic, Central European, Greek, Turkish, Hebrew, Arabic, Baltic, Vietnamese, Thai (was Windows-1252 only). Pasted from the system clipboard where the platform offers it (Windows, Wayland) when there is no HTML (#687). Thin: comments, notes, revisions, sections, RTL | | `.odt` OpenDocument Text | 2% | R/W | 🟡 | 🟡 | ~50% | 8–12 | `crates/formats/src/tests.rs` | Styles, lists (#87), tables, pictures, headers/footers, page breaks (#86), page size, orientation and margins (#810). No footnotes, comments, tracked changes, fields or RTL | -| `.txt` Plain text | 2% | R/W | ✅ | ✅ | ~95% | 1–2 | unit tests | Encoding detection. Files with other extensions (`.m3u`/`.m3u8` playlists, logs, config files, no extension) open as text when they are text, Word and RTF files by their signature; binary files are refused (#569). Word's encoding choice dialog and line-ending options missing; such files save only through Save As | +| `.txt` Plain text | 2% | R/W | ✅ | ✅ | ~95% | 1–2 | unit tests | Encoding detection. Files with other extensions (`.m3u`/`.m3u8` playlists, logs, config files, no extension) open as text when they are text, Word and RTF files by their signature; binary files are refused (#569). A Word or RTF extension that contradicts the content also opens by signature, as in Word: RTF saved as `.doc` (hh.ru résumés, accounting exports), a `.docx` renamed `.doc`, a Word 97-2003 file renamed `.docx`. Word's encoding choice dialog and line-ending options missing; such files save only through Save As | | `.htm/.html` Web Page and Web Page, Filtered | 1% | R/W | 🟡 | 🟡 | ~60% | 6–10 | `crates/formats/src/tests.rs` | Tables (#102), relative pictures embedded (#113), missing `` tolerated (#154). No Word "Web Page" (with `_files` folder) round-trip. The same reader takes HTML from the system clipboard: Paste keeps another program's formatting (a Word document, a web page; Windows' "HTML Format" header understood), Merge Formatting keeps its emphasis (#687) | | `.mht/.mhtml` Single File Web Page | <1% | R/W | ❌ | ❌ | 0% | 3–5 | — | | | `.xml` Word XML Document (Flat OPC) | <1% | R/W | ✅ | ✅ | ~65% | — | `crates/docx/tests/word_xml.rs`, `roundtrip.rs` (`flat_opc_round_trip_equals_docx`) | The DOCX package in one XML file, read and written by the DOCX code (#548); File › Export › Word XML Document. `.xml` files are told apart by their root element | @@ -165,6 +165,7 @@ raise readiness estimates. No Word-derived files are fixtures. |---|---|---| | 2026-10-12 | trivial | Paste keeps the formatting of HTML and RTF on the system clipboard; Paste Special lists them and pictures (#687, #833) | | 2026-10-12 | trivial | ODT page size, orientation and margins read back from `styles.xml` on open, so they survive a save and reopen (#810) | +| 2026-10-12 | minor | RTF reads 8-bit text in the document's or the font's code page (`\ansicpg`, `\fcharset`, `\cpg`) instead of always Windows-1252; a `.doc`/`.docx`/`.rtf` whose content is another of these formats opens by signature. Found on a private corpus of real Russian documents: 38 of 44 `.doc` files were RTF, all now open with their Cyrillic text | | 2026-10-12 | trivial | `w:sym` and symbol-font runs keep their stored characters on save; TXT export and copy write their Unicode equivalents (#776) | | 2026-10-12 | trivial | Insert › Picture, drag-and-drop and paste take Windows Metafiles and Enhanced Metafiles (#802); new row for picture formats | | 2026-10-11 | trivial | Independent complex-script size, bold and italic inherit correctly through defaults and styles; explicit equal values survive; six synthetic multi-generation round-trip tests, including tracked formatting, added |