"""Answer key: six properties, each one checkable in a line."""
S = "Aéż中😀"

print("1. AN ASCII FILE ALREADY IS A UTF-8 FILE")
print(f'   "Hello".encode("ascii") == "Hello".encode("utf-8") -> '
      f'{"Hello".encode("ascii") == "Hello".encode("utf-8")}')
print("   Not a conversion, not a compatibility mode -- the same bytes. Every")
print("   ASCII document ever written was already valid UTF-8 on the day the")
print("   encoding was designed, which is what made adoption free.")
print()
print("2. NO FRAGMENT OF A CHARACTER CAN LOOK LIKE ASCII")
high = [b for c in S if ord(c) > 127 for b in c.encode()]
print(f"   every byte of every non-ASCII character here: "
      f"{'all >= 0x80' if all(b >= 0x80 for b in high) else 'SOME ARE ASCII'}")
print("   So a byte below 0x80 is always that ASCII character and never part of")
print("   something else. grep for a slash and you cannot match half a Chinese")
print("   character -- which is exactly the bug overlong sequences reintroduce.")
print()
print("3. YOU CAN START ANYWHERE")
b = S.encode()
def is_lead(x): return not (0x80 <= x < 0xC0)
print(f"   bytes {b.hex(' ')}")
print(f"   lead bytes at offsets {[i for i, x in enumerate(b) if is_lead(x)]}")
print("   A continuation byte is 10xxxxxx and a lead byte never is, so from any")
print("   offset you can walk backwards at most three bytes to find the start")
print("   of a character. UTF-16 has no such rule mid-stream, and a code page")
print("   with shift states has none at all.")
print()
print("4. A WRONG GUESS FAILS LOUDLY")
try:
    b"\xe9\xe9\xe9".decode("utf-8")
except UnicodeDecodeError:
    print("   b'\\xe9\\xe9\\xe9'.decode('utf-8') -> UnicodeDecodeError")
print(f"   the same bytes as latin-1      -> {b'\\xe9\\xe9\\xe9'.decode('latin-1')!r}")
print("   Latin-1 accepts every byte, so a wrong guess there is silent forever.")
print("   UTF-8's structure means most wrong guesses produce invalid sequences")
print("   quickly -- which is the property that makes detection possible at all.")
print()
print("5. THERE IS NO BYTE ORDER TO GET WRONG")
print(f"   utf-8    {S[:2].encode('utf-8').hex(' ')}")
print(f"   utf-16le {S[:2].encode('utf-16-le').hex(' ')}")
print(f"   utf-16be {S[:2].encode('utf-16-be').hex(' ')}")
print("   The unit is one byte, so there is nothing to order. No BOM is needed,")
print("   and the three bytes some tools write are a signature, not a mark.")
print()
print("6. SORTING THE BYTES SORTS THE CHARACTERS")
chars = sorted("Aéż中😀")
by_bytes = sorted("Aéż中😀", key=lambda c: c.encode())
print(f"   by code point {chars}")
print(f"   by utf-8 byte {by_bytes}")
print(f"   same order? {chars == by_bytes}")
print("   True for UTF-8 and NOT for UTF-16, where the surrogate range sits")
print("   below U+FFFF numerically but encodes characters above it -- so a")
print("   UTF-16 byte sort puts every emoji in the middle of the BMP.")
print()
print("WHAT IT PAYS")
for text, label in [("hello world", "English"), ("中文字符测试", "Chinese")]:
    u8, u16 = len(text.encode("utf-8")), len(text.encode("utf-16-le"))
    print(f"   {label:<8} utf-8 {u8:>3} bytes   utf-16 {u16:>3} bytes   "
          f"{'utf-8 wins' if u8 < u16 else 'UTF-16 IS SMALLER'}")
print("   CJK text costs 3 bytes per character in UTF-8 and 2 in UTF-16, so")
print("   the bill is real and it is paid by the people who write in those")
print("   scripts. That argument lost -- to the six properties above, and to")
print("   the fact that most bytes on the wire are markup, which is ASCII.")

assert "Hello".encode("ascii") == "Hello".encode("utf-8")
assert sorted("Aéż中😀") == sorted("Aéż中😀", key=lambda c: c.encode())