WhatsApp Binary Protocol

wacore/binary/src/
├── marshal.rs     # Serialization entry points
├── encoder.rs     # Binary encoding logic
├── decoder.rs     # Binary decoding logic
├── node.rs        # Node data structures
├── token.rs       # Token dictionary
├── jid.rs         # JID (identifier) handling
└── builder.rs     # Fluent API for node construction

use compact_str::CompactString;
use std::borrow::Cow;

pub struct Node {
    pub tag: Cow<'static, str>,        // e.g., "message", "receipt", "iq"
    pub attrs: Attrs,                  // Key-value attributes
    pub content: Option<NodeContent>,  // Optional content
}

pub enum NodeContent {
    Bytes(Vec<u8>),             // Binary payload
    String(CompactString),      // Text payload
    Nodes(Vec<Node>),           // Child nodes
}

use compact_str::CompactString;

pub enum NodeValue {
    String(CompactString),
    Jid(Jid),                // Optimized for WhatsApp identifiers
}

pub struct Attrs(pub Vec<(Cow<'static, str>, NodeValue)>);

use std::borrow::Cow;

// Get a string view of the value (works for both variants)
// - String variant: Cow::Borrowed(&str) — zero copy
// - Jid variant: Cow::Owned(formatted) — allocates only when needed
pub fn as_str(&self) -> Cow<'_, str>

// Convert to an owned Jid, parsing from string if necessary
// - Jid variant: clones the Jid directly
// - String variant: attempts to parse, returns None on failure
pub fn to_jid(&self) -> Option<Jid>

// Reading an attribute value as a string
let msg_type = node.attrs.get("type")
    .map(|v| v.as_str().into_owned());

// Reading an attribute value as a JID
let recipient = node.attrs.get("to")
    .and_then(|v| v.to_jid());

use compact_str::CompactString;

pub struct Jid {
    pub user: CompactString,           // "15551234567"
    pub server: Server,                // Server::Pn (s.whatsapp.net)
    pub agent: u8,                     // Agent byte (parsed from JID string)
    pub device: u16,                   // Device ID (0 for primary)
    pub integrator: u16,               // Integrator ID (used with interop server)
}

#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, PartialOrd, Ord, Hash)]
#[repr(u8)]
pub enum Server {
    #[default]
    Pn = 0,          // s.whatsapp.net — Standard phone-number JIDs
    Lid = 1,         // lid — Linked Identity JIDs
    Group = 2,       // g.us — Group chat JIDs
    Broadcast = 3,   // broadcast — Broadcast lists and status
    Newsletter = 4,  // newsletter — Newsletter / channel JIDs
    Hosted = 5,      // hosted — Cloud API business devices (phone-based)
    HostedLid = 6,   // hosted.lid — Cloud API business devices (LID-based)
    Messenger = 7,   // msgr — Messenger interop JIDs
    Interop = 8,     // interop — Cross-platform interop JIDs
    Bot = 9,         // bot — Bot JIDs
    Legacy = 10,     // c.us — Legacy user server (pre-multidevice)
}

let server = Server::Pn;
assert_eq!(server.as_str(), "s.whatsapp.net");
assert!(server == "s.whatsapp.net");  // PartialEq<str> for backward compat

let parsed = Server::try_from("g.us").unwrap();
assert_eq!(parsed, Server::Group);

Jid::pn("15551234567")            // @s.whatsapp.net, device 0
Jid::lid("ABC123")                // @lid, device 0
Jid::group("12345678")            // @g.us
Jid::newsletter("12345678")       // @newsletter
Jid::pn_device("15551234567", 1)  // @s.whatsapp.net, device 1
Jid::lid_device("ABC123", 2)      // @lid, device 2
Jid::status_broadcast()           // status@broadcast
Jid::new("user", Server::Pn)      // arbitrary server variant

use wacore_binary::jid::{Jid, JidExt};

let jid = Jid::pn("15551234567");
assert!(jid.is_ad());        // true — s.whatsapp.net is an AD server
assert!(!jid.is_group());    // false — not a @g.us JID

use wacore_binary::builder::NodeBuilder;
use wacore_binary::jid::Jid;

let message = NodeBuilder::new("message")
    .attr("to", "15551234567@s.whatsapp.net")
    .attr("type", "text")
    .attr("id", "ABCD1234")
    .children(vec![
        NodeBuilder::new("body").string_content("Hello, world!").build(),
    ])
    .build();

// Prefer jid_attr for JID attributes — avoids string allocation
let receipt = NodeBuilder::new("receipt")
    .attr("id", &message_id)
    .jid_attr("to", chat_jid.clone())
    .jid_attr("participant", sender_jid.clone())
    .build();

// Equivalent but allocates a string per JID
let receipt = NodeBuilder::new("receipt")
    .attr("id", &message_id)
    .attr("to", chat_jid.to_string())
    .attr("participant", sender_jid.to_string())
    .build();

let mut builder = NodeBuilder::new("receipt")
    .attr("id", &info.id)
    .jid_attr("to", info.source.chat.clone());

if info.category == MessageCategory::Peer {
    builder = builder.attr("type", "peer_msg");
}

if info.source.is_group {
    builder = builder.jid_attr("participant", info.source.sender.clone());
}

let node = builder.build();

// Single-byte tokens (4-235)
pub const LIST_EMPTY: u8 = 0;
pub const INTEROP_JID: u8 = 245; // Interop JID
pub const FB_JID: u8 = 246;      // Facebook JID
pub const AD_JID: u8 = 247;      // JID with device ID
pub const LIST_8: u8 = 248;      // List with <256 items
pub const LIST_16: u8 = 249;     // List with ≥256 items
pub const JID_PAIR: u8 = 250;    // JID in user@server format
pub const HEX_8: u8 = 251;       // Packed hex string
pub const BINARY_8: u8 = 252;    // Binary data <256 bytes
pub const BINARY_20: u8 = 253;   // Binary data <1MB
pub const BINARY_32: u8 = 254;   // Binary data ≥1MB
pub const NIBBLE_8: u8 = 255;    // Packed numeric string

use wacore_binary::token::{index_of_token, TokenKind};

index_of_token("message") => Some(TokenKind::Single(19))
index_of_token("iq")      => Some(TokenKind::Single(18))
index_of_token("body")    => Some(TokenKind::Single(7))
index_of_token("participant") => Some(TokenKind::Double(dict, idx))
index_of_token("unknown_string") => None

pub enum TokenKind {
    Single(u8),
    Double(u8, u8),  // (dictionary index, token index)
}

get_single_token(19)    => Some("message")
get_double_token(0, 42) => Some("participant")

// Basic serialization
pub fn marshal(node: &Node) -> Result<Vec<u8>>

// Serialize to existing buffer (zero-copy for output)
pub fn marshal_to_vec(node: &Node, output: &mut Vec<u8>) -> Result<()>

// Two-pass encoding with exact size pre-calculation
pub fn marshal_exact(node: &Node) -> Result<Vec<u8>>

// Auto-sizing with heuristics
pub fn marshal_auto(node: &Node) -> Result<Vec<u8>>

enum StringHint {
    Empty,                          // "" → BINARY_8 + 0
    SingleToken(u8),                // "message" → 19
    DoubleToken { dict: u8, token: u8 },
    PackedNibble,                   // "123-456" → compressed
    PackedHex,                      // "DEADBEEF" → compressed
    Jid(ParsedJidMeta),             // JID-specific encoding
    RawBytes,                       // Fallback
}

// Input: "123-456.789"
// Encoding:
// '1' → 1, '2' → 2, '3' → 3, '-' → 10, '4' → 4, ...
// Packed: 0x12, 0x3A, 0x45, 0x67, 0x89

pub const PACKED_MAX: u8 = 127;  // Max length for packed/token strings

fn pack_nibble(value: u8) -> u8 {
    match value {
        b'-' => 10,
        b'.' => 11,
        0 => 15,  // Padding
        c if c.is_ascii_digit() => c - b'0',
        _ => panic!("Invalid nibble"),
    }
}

// Input: "DEADBEEF"
// Packed: 0xDE, 0xAD, 0xBE, 0xEF

fn pack_hex(value: u8) -> u8 {
    match value {
        c if c.is_ascii_digit() => c - b'0',
        c if (b'A'..=b'F').contains(&c) => 10 + (c - b'A'),
        0 => 15,  // Padding
        _ => panic!("Invalid hex"),
    }
}

while input_bytes.len() >= 16 {
    let input = u8x16::from_slice(chunk);
    let indices = input.saturating_sub(nibble_base);
    let nibbles = lookup.swizzle_dyn(indices);
    
    let (evens, odds) = nibbles.deinterleave(
        nibbles.rotate_elements_left::<1>()
    );
    let packed = (evens << Simd::splat(4)) | odds;
    self.write_raw_bytes(&packed.to_array()[..8])?;
}

// Format: JID_PAIR + user + server
// Example: "15551234567@s.whatsapp.net"
self.write_u8(token::JID_PAIR)?;
if user.is_empty() {
    self.write_u8(token::LIST_EMPTY)?;
} else {
    self.write_string(user)?;  // "15551234567"
}
self.write_string(server)?;    // "s.whatsapp.net"

// Format: AD_JID + domain_type + device + user
// Example: "15551234567:1@s.whatsapp.net" (device 1)
self.write_u8(token::AD_JID)?;
self.write_u8(server_to_domain_type(jid.server, jid.agent))?;
self.write_u8(device)?;            // Device number
self.write_string(user)?;          // User part only

fn write_list_start(&mut self, len: usize) -> Result<()> {
    if len == 0 {
        self.write_u8(token::LIST_EMPTY)?;  // 0x00
    } else if len < 256 {
        self.write_u8(token::LIST_8)?;      // 0xF8
        self.write_u8(len as u8)?;
    } else {
        self.write_u8(token::LIST_16)?;     // 0xF9
        self.write_u16_be(len as u16)?;
    }
    Ok(())
}

LIST_START(list_len)
    tag
    attr_key_1
    attr_value_1
    attr_key_2
    attr_value_2
    ...
    [content]  // If present

pub fn write_node<N: EncodeNode>(&mut self, node: &N) -> Result<()> {
    let content_len = if node.has_content() { 1 } else { 0 };
    let list_len = 1 + (node.attrs_len() * 2) + content_len;
    
    self.write_list_start(list_len)?;
    self.write_string(node.tag())?;
    node.encode_attrs(self)?;
    node.encode_content(self)?;
    Ok(())
}

pub struct Decoder<'a> {
    data: &'a [u8],
    offset: usize,
}

impl<'a> Decoder<'a> {
    pub fn read_node_ref(&mut self) -> Result<NodeRef<'a>>
    pub fn read_list_size(&mut self) -> Result<usize>
    pub fn read_string(&mut self, len: usize) -> Result<NodeStr<'a>>
}

/// Borrowed-or-inline string for decoded nodes.
pub enum NodeStr<'a> {
    Borrowed(&'a str),        // Zero-copy: points into input buffer
    Owned(CompactString),     // Inline for short strings (≤24 bytes)
}

pub struct NodeRef<'a> {
    pub tag: NodeStr<'a>,          // Borrowed or inline
    pub attrs: AttrsRef<'a>,       // Vec<(NodeStr<'a>, ValueRef<'a>)>
    pub content: Option<Box<NodeContentRef<'a>>>,
}

pub enum NodeContentRef<'a> {
    Bytes(Cow<'a, [u8]>),    // Zero-copy for byte content
    String(NodeStr<'a>),     // Borrowed or inline
    Nodes(Box<NodeVec<'a>>), // Recursive borrowing
}

pub struct OwnedNodeRef {
    inner: Yoke<NodeRef<'static>, Vec<u8>>,
}

impl OwnedNodeRef {
    /// Decode a node from an owned buffer.
    pub fn new(buffer: Vec<u8>) -> Result<Self>;

    /// Access the borrowed node.
    pub fn get(&self) -> &NodeRef<'_>;

    /// Convert to an owned Node (allocates — use sparingly).
    pub fn to_owned_node(&self) -> Node;

    // Convenience accessors: tag(), attrs(), get_attr(),
    // children(), get_optional_child(), content_bytes(), etc.
}

use serde_json;

// Serialize an OwnedNodeRef directly — zero-copy from the network buffer
let owned_ref: OwnedNodeRef = OwnedNodeRef::new(buffer)?;
let json = serde_json::to_string(&owned_ref)?;

// Equivalent but allocates: convert to Node first
let json_owned = serde_json::to_string(&owned_ref.to_owned_node())?;

// Both produce identical JSON output
assert_eq!(json, json_owned);

pub enum ValueRef<'a> {
    String(NodeStr<'a>),
    Jid(JidRef<'a>),
}

let mut parser = node.attr_parser();

let msg_id: String = parser.required_str("id");
let msg_type: Option<String> = parser.optional_str("type");
let recipient: Option<Jid> = parser.optional_jid("to");

parser.finish()?; // Returns accumulated errors if any

fn unpack_nibble(packed: u8, position: u8) -> u8 {
    let nibble = if position == 0 {
        (packed >> 4) & 0x0F
    } else {
        packed & 0x0F
    };
    
    match nibble {
        0..=9 => b'0' + nibble,
        10 => b'-',
        11 => b'.',
        15 => 0,  // Padding
        _ => panic!("Invalid nibble"),
    }
}

fn intern_cow(s: &str) -> Cow<'static, str> {
    if let Some(kind) = token::index_of_token(s) {
        let interned = match kind {
            token::TokenKind::Single(idx) => token::get_single_token(idx),
            token::TokenKind::Double(dict, idx) => token::get_double_token(dict, idx),
        };
        if let Some(token) = interned {
            return Cow::Borrowed(token);    // Zero-alloc: points to static str
        }
    }
    Cow::Owned(s.to_string())               // Fallback: heap-allocate unknown strings
}

pub fn marshal_exact(node: &Node) -> Result<Vec<u8>> {
    // Pass 1: Calculate exact size
    let plan = build_marshaled_node_plan(node);
    
    // Pass 2: Encode directly into fixed-size buffer
    let mut payload = vec![0; plan.size];
    let mut encoder = Encoder::new_slice(&mut payload, Some(&plan.hints))?;
    encoder.write_node(node)?;
    Ok(payload)
}

pub struct StringHintCache {
    hints: Vec<(StrKey, StringHint)>,
}

impl StringHintCache {
    fn hint_or_insert(&mut self, s: &str) -> StringHint {
        // Strings longer than PACKED_MAX can't be tokens,
        // packed nibble/hex, or JIDs — skip classification entirely
        if s.len() > token::PACKED_MAX as usize {
            return StringHint::RawBytes;
        }
        if let Some(existing) = self.hints.iter().find(...) {
            return existing;
        }
        let hint = classify_string_hint(s);
        self.hints.push((key, hint));
        hint
    }
}

fn estimate_capacity_node(node: &Node) -> usize {
    let mut estimate = DEFAULT_MARSHAL_CAPACITY + 16;
    estimate += node.tag.len();  // Works with both Cow::Borrowed and Cow::Owned
    estimate += node.attrs.len() * AUTO_ATTR_ESTIMATE;  // ~24 bytes/attr
    
    if let Some(NodeContent::Nodes(children)) = &node.content {
        estimate += children.len() * AUTO_CHILD_ESTIMATE;  // ~96 bytes/child
        
        // Sample first 32 children for better accuracy
        for child in children.iter().take(AUTO_CHILD_SAMPLE_LIMIT) {
            estimate += child.tag.len() + ...
        }
    }
    
    estimate.clamp(DEFAULT_MARSHAL_CAPACITY, AUTO_MAX_HINT_CAPACITY)
}

// Request
NodeBuilder::new("iq")
    .attr("id", "ABC123")
    .attr("type", "get")
    .attr("xmlns", "w:g2")
    .attr("to", "@s.whatsapp.net")
    .children(vec![
        NodeBuilder::new("query").build(),
    ])
    .build()

// Response
NodeBuilder::new("iq")
    .attr("id", "ABC123")
    .attr("type", "result")
    .attr("from", "@s.whatsapp.net")
    .children(vec![
        NodeBuilder::new("group")
            .attr("id", "123456@g.us")
            .attr("subject", "My Group")
            .build(),
    ])
    .build()

NodeBuilder::new("message")
    .attr("to", "15551234567@s.whatsapp.net")
    .attr("type", "text")
    .attr("id", message_id)
    .children(vec![
        NodeBuilder::new("enc")
            .attr("v", "2")
            .attr("type", "msg")
            .bytes(encrypted_payload)
            .build(),
    ])
    .build()

NodeBuilder::new("receipt")
    .attr("to", "15551234567@s.whatsapp.net")
    .attr("id", message_id)
    .attr("type", "read")
    .attr("t", timestamp)
    .build()

Node: <message type="text"/>

Binary:
  F8 03           LIST_8(3)  [tag + 2 attrs]
  13              Token("message")
  16              Token("type")
  07              Token("text")

Node: <message type="text"><body>Hi</body></message>

Binary:
  F8 04           LIST_8(4)  [tag + 2 attrs + content]
  13              Token("message")
  16              Token("type")
  07              Token("text")
  F8 02           LIST_8(2)  [child: tag + content]
  07              Token("body")
  FC 02           BINARY_8(2)
  48 69           "Hi"

:dep wacore-binary = { path = "wacore/binary" }
:dep hex = "0.4"

use wacore_binary::marshal::unmarshal_ref;
use wacore_binary::builder::NodeBuilder;

// Decode binary data
{
    let data = hex::decode("f8034c1a07").unwrap();
    let node = unmarshal_ref(&data).unwrap();
    println!("Tag: {}", node.tag);
    for (k, v) in node.attrs.iter() {
        println!("  {}: {}", k, v);
    }
}

// Encode and inspect
{
    let node = NodeBuilder::new("message")
        .attr("type", "text")
        .build();
    let bytes = marshal(&node).unwrap();
    println!("Encoded: {:02x?}", bytes);
}

pub enum BinaryError {
    UnexpectedEof,
    InvalidToken(u8),
    InvalidListSize,
    AttrParse(String),
    Jid(JidParseError),     // JID parse failures from AttrParser
    LeftoverData(usize),
    Io(std::io::Error),
}