raw derive: a placeholder text/plain no longer displaces the HTML text of a mail (#204)

A mail's text is its plain text unless that is under 10% of the HTML text, then the HTML text
with a warning; one rule for .eml and .msg. The other version of the text in a
multipart/alternative is no longer an attachment.

Files changed:
- CHANGES.md
- VERSION
- docs/reference/commands/raw-material-and-uploads.md
- testfiles/mail/gog-roundcube.eml.expect
- tools/CONTRACT.md
- tools/chemenu/commands/raw_cmd.py
- tools/chemenu/filters/mail.py
- tools/chemenu/filters/msg.py
- tools/chemenu/tests/test_raw_derive.py
- tools/chemenu/tests/test_raw_derive_binary.py

Co-Authored-By: Claude Code <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01SnAJ7Z3CpVD3PRbN73QtU2

tools/chemenu/tests/test_raw_derive.py

@@ -182,8 +182,8 @@ def _text_cell(ref: str, text: str) -> str:
     return f'<c r="{ref}" t="inlineStr"><is><t xml:space="preserve">{text.replace(chr(13), "&#13;")}</t></is></c>'
 
 
-def make_eml(body: str = "Hallo Welt\n", *, html: str = "", attach: bool = False, inline: bool = False,
-             subject: str = "Angebot") -> bytes:
+def make_eml(body: str = "Hallo Welt\n", *, html: str = "", alternative: str = "", attach: bool = False,
+             inline: bool = False, subject: str = "Angebot") -> bytes:
     msg = EmailMessage()
     msg["From"] = "Anna Beispiel <anna@example.org>"
     msg["To"] = "bob@example.org, Carla <carla@example.org>"
@@ -194,6 +194,8 @@ def make_eml(body: str = "Hallo Welt\n", *, html: str = "", attach: bool = False
         msg.set_content(html, subtype="html")
     else:
         msg.set_content(body)
+    if alternative:
+        msg.add_alternative(alternative, subtype="html")
     if inline:
         msg.add_related(b"\x89PNG", maintype="image", subtype="png", cid="<logo@x>")
     if attach:
@@ -752,6 +754,87 @@ def test_eml_with_inline_image_warns_and_is_not_an_attachment(tree, capsys):
     assert "inline image" in capsys.readouterr().out
 
 
+_NEWSLETTER = "<html><body><h1>Newsletter</h1><p>" + "Real content of the mail. " * 20 + "</p></body></html>"
+
+
+def _eml_parsed(data: bytes):
+    from chemenu.filters import mail
+
+    return mail.parse(data)
+
+
+def test_eml_placeholder_plain_text_gives_way_to_the_html_text(tree, capsys):
+    _put(tree, "mail.eml", make_eml("Plain text version not available", alternative=_NEWSLETTER))
+    _derive("incoming/mail.eml")
+    out = capsys.readouterr().out
+    header, text = _header_and_text(tree / "incoming/mail/derived.md")
+
+    assert "Real content of the mail." in text and "not available" not in text
+    assert "The plain text (32 characters) is under 10% of the HTML text" in out
+    assert header["attachments"] == []
+    assert sorted(p.name for p in (tree / "incoming/mail").iterdir()) == ["derived.md", "original.eml"]
+
+
+def test_eml_good_plain_text_wins_over_html_without_a_warning(tree, capsys):
+    plain = "Real content of the mail. " * 18
+    _put(tree, "mail.eml", make_eml(plain, alternative=_NEWSLETTER))
+    _derive("incoming/mail.eml")
+    out = capsys.readouterr().out
+    header, text = _header_and_text(tree / "incoming/mail/derived.md")
+
+    assert text.strip() == plain.strip()
+    assert "is under" not in out
+    assert header["attachments"] == []
+
+
+def test_eml_keeps_a_real_attachment_beside_the_two_text_versions(tree):
+    _put(tree, "mail.eml", make_eml("Plain text version not available", alternative=_NEWSLETTER, attach=True))
+    _derive("incoming/mail.eml")
+    header, _text = _header_and_text(tree / "incoming/mail/derived.md")
+
+    assert header["attachments"] == ["a.bin"]
+    assert (tree / "incoming/mail/a.bin").read_bytes() == b"data"
+
+
+def test_eml_html_inside_a_related_group_is_still_a_text_version_and_its_image_is_inline():
+    msg = EmailMessage()
+    msg["Subject"] = "S"
+    msg.set_content("Plain text version not available")
+    msg.add_alternative(_NEWSLETTER, subtype="html")
+    msg.get_payload()[1].add_related(b"\x89PNG", maintype="image", subtype="png", cid="<logo@x>")
+    parsed = _eml_parsed(msg.as_bytes())
+
+    assert "Real content of the mail." in parsed.body
+    assert parsed.attachments == []
+    assert parsed.inline_images == 1
+
+
+def test_eml_text_version_marked_as_an_attachment_stays_one():
+    msg = EmailMessage()
+    msg["Subject"] = "S"
+    msg.set_content("Hello there\n")
+    msg.add_alternative(_NEWSLETTER, subtype="html")
+    msg.get_payload()[1].add_header("Content-Disposition", "attachment", filename="version.html")
+    parsed = _eml_parsed(msg.as_bytes())
+
+    assert parsed.body == "Hello there"
+    assert [a.name for a in parsed.attachments] == ["version.html"]
+
+
+def test_eml_plain_and_html_in_a_mixed_mail_without_alternative_keep_the_html_as_attachment():
+    msg = EmailMessage()
+    msg["Subject"] = "S"
+    msg.set_content("Hello there. " * 40)
+    msg.make_mixed()
+    html = EmailMessage()
+    html.set_content(_NEWSLETTER, subtype="html")
+    msg.attach(html)
+    parsed = _eml_parsed(msg.as_bytes())
+
+    assert parsed.body == ("Hello there. " * 40).strip()
+    assert len(parsed.attachments) == 1
+
+
 # --- the registry -----------------------------------------------------------
 
 

tools/chemenu/tests/test_raw_derive_binary.py

@@ -165,6 +165,32 @@ def test_a_msg_with_only_html_takes_the_html_text(tree):
     assert "Only" in text and "html here" in text and "<p>" not in text
 
 
+_HTML_BODY = "<html><body><p>" + "Real content of the mail. " * 20 + "</p></body></html>"
+
+
+@needs_olefile
+def test_a_msg_with_a_placeholder_plain_text_takes_the_html_text_and_warns(tree, capsys):
+    _put(tree, "p.msg", build_msg(subject="S", body="Plain text version not available", html=_HTML_BODY))
+    _derive("incoming/p.msg")
+    out = capsys.readouterr().out
+    _header, text = _header_and_text(tree / "incoming/p/derived.md")
+
+    assert "Real content of the mail." in text and "not available" not in text
+    assert "The plain text (32 characters) is under 10% of the HTML text" in out
+
+
+@needs_olefile
+def test_a_msg_with_a_good_plain_text_keeps_it_without_a_warning(tree, capsys):
+    plain = "Real content of the mail. " * 18
+    _put(tree, "g.msg", build_msg(subject="S", body=plain, html=_HTML_BODY))
+    _derive("incoming/g.msg")
+    out = capsys.readouterr().out
+    _header, text = _header_and_text(tree / "incoming/g/derived.md")
+
+    assert text.strip() == plain.strip()
+    assert "is under" not in out
+
+
 @needs_olefile
 def test_a_msg_with_only_rtf_warns_and_has_no_text(tree, capsys):
     _put(tree, "r.msg", build_msg(subject="S", rtf=b"{\\rtf1 hello}"))