module Uutf:sig..end
  Uutf is a non-blocking streaming codec to decode and
  encode the 
  UTF-8,  UTF-16, UTF-16LE
  and UTF-16BE encoding schemes. It can efficiently work character by
  character without blocking on IO. Decoders perform
  character position tracking and support newline normalization.
  Functions are also provided to  fold over the
  characters of UTF encoded OCaml string values and to
  directly encode characters in OCaml Buffer.t
  values.
See examples of use.
Release 0.9.4 - Daniel Bünzli <daniel.buenzl i@erratique.ch>
      Uutf uses the term character for a Unicode
       scalar
      value which is an integer value in the ranges 0x0000
      ... 0xD7FF and 0xE000 ... 0x10FFFF. This should not be
      confused with a Unicode
      code point, which is
      a scalar value or a (textually meaningless)
      surrogate
      code point.
typeuchar =int
val u_bom : uchar
val u_rep : uchar
val is_uchar : int -> bool
val cp_to_string : int -> stringcp_to_string cp represents the
    code point cp in
    ASCII according to the Unicode notational convention
    (see Appendix A in Unicode 6.1.0).
    If cp is not a valid code point "U+Invalid(X)" is
    returned where X is the hexadecimal integer value.
    Warning. Not thread safe. Use Uutf.pp_cp for thread safety.
val pp_cp : Format.formatter -> int -> unit
typeencoding =[ `UTF_16 | `UTF_16BE | `UTF_16LE | `UTF_8 ]
typedecoder_encoding =[ `ISO_8859_1 | `US_ASCII | `UTF_16 | `UTF_16BE | `UTF_16LE | `UTF_8 ]
Uutf. Unicode encoding
    schemes plus US-ASCII and
    
    ISO/IEC 8859-1 (latin-1).val encoding_of_string : string -> decoder_encoding option
val encoding_to_string : [< decoder_encoding ] -> string
typesrc =[ `Channel of Pervasives.in_channel | `Manual | `String of string ]
`Manual source the client
    must provide input with Uutf.Manual.src.typenln =[ `ASCII of uchar | `NLF of uchar | `Readline of uchar ]
`ASCII, normalizes CR (U+000D), LF (U+000A) and CRLF
       (<U+000D, U+000A>).`NLF, normalizes the Unicode newline function (NLF). This is
       NEL (U+0085) and the normalizations of `ASCII.`Readline, normalizes for a Unicode readline function. This is FF
       (U+000C), LS (U+2028), PS (U+2029), and the normalizations
       of `NLF.`NLF and
    `Readline normalizations allow to implement all the different
    recommendations of Unicode's newline guidelines (section 5.8 in
    Unicode 6.1.0).type 
val decoder : ?nln:[< nln ] ->
       ?encoding:[< decoder_encoding ] -> [< src ] -> decoderdecoder nln encoding src is a decoder that inputs from src.
    Byte order mark.
    Byte order mark
    (BOM) constraints are application dependent and prone to
    misunderstandings (see the
    FAQ). Hence,
    Uutf decoders have a simple rule: an initial BOM is always
    removed from the input and not counted in character position
    tracking. The function Uutf.decoder_removed_bom does however return
    true if a BOM was removed so that all the information can be
    recovered if needed.
    For UTF-16BE and UTF-16LE the above rule is a violation of
    conformance D96 and D97 of the standard. Uutf favors the idea
    that if there's a BOM, decoding with `UTF_16 or the `UTF_16XX
    corresponding to the BOM should decode the same character sequence
    (this is not the case if you stick to the standard). The client
    can however regain conformance by consulting the result of
    Uutf.decoder_removed_bom and take appropriate action.
    Encoding. encoding specifies the decoded encoding
    scheme. If `UTF_16 is used the endianness is determined
    according to the standard: from a
    BOM
    if there is one, `UTF_16BE otherwise.
    If encoding is unspecified it is guessed. The result of a guess
    can only be `UTF_8, `UTF_16BE or `UTF_16LE. The heuristic
    looks at the first three bytes of input (or less if impossible)
    and takes the first matching byte pattern in the table below.
xx = any byte
.. = any byte or no byte (input too small)
pp = positive byte
uu = valid UTF-8 first byte
Bytes    | Guess     | Rationale
---------+-----------+-----------------------------------------------
EF BB BF | `UTF_8    | UTF-8 BOM
FE FF .. | `UTF_16BE | UTF-16BE BOM
FF FE .. | `UTF_16LE | UTF-16LE BOM
00 pp .. | `UTF_16BE | ASCII UTF-16BE and U+0000 is often forbidden
pp 00 .. | `UTF_16LE | ASCII UTF-16LE and U+0000 is often forbidden
uu .. .. | `UTF_8    | ASCII UTF-8 or valid UTF-8 first byte.
xx xx .. | `UTF_16BE | Not UTF-8 => UTF-16, no BOM => UTF-16BE
.. .. .. | `UTF_8    | Single malformed UTF-8 byte or no input.
Uutf.decoder_removed_bom will tell the client
    if the guess was BOM based.
    Newline normalization. If nln is specified, the given
    newline normalization is performed, see Uutf.nln. Otherwise
    all newlines are returned as found in the input.
    Character position. The line number, column number and
    character count of the last decoded character (including
    `Malformed ones) are respectively returned by Uutf.decoder_line,
    Uutf.decoder_col and Uutf.decoder_count. Before the first call to
    Uutf.decode the line number is 1 and the column is 0.  Each
    Uutf.decode returning `Uchar or `Malformed increments the column
    until a newline.  On a newline, the line number is incremented and
    the column set to zero. For example the line is 2 and column 0
    after the first newline was decoded. This can be understood as if Uutf.decode
    was moving an insertion point to the right in the data.  A     newline is anything normalized by `Readline, see Uutf.nln.
    Uutf assumes that each Unicode scalar value has a column width
    of 1. The same assumption may not be made by the display program
    (e.g. for emacs' compilation mode you need to set
    compilation-error-screen-columns to nil). For implementing
    more involved column width increments yourself, look into
    wcwidth and
    
    grapheme cluster boundaries.
val decode : decoder ->
       [ `Await | `End | `Malformed of string | `Uchar of uchar ]decode d is:
    `Await if d has a `Manual input source and awaits
       for more input. The client must use Uutf.Manual.src to provide it.`Uchar u if a Unicode scalar value u was decoded.`End if the end of input was reached.`Malformed bytes if the bytes sequence is malformed according to
       the decoded encoding scheme. If you are interested in a best-effort
       decoding you can still continue to decode after an error until the
       decoder synchronizes again on valid bytes. It may however be a good
       idea to signal the malformed characters by adding an Uutf.u_rep
       character to the parsed data, see the examples.
    Note. Repeated invocation always eventually returns `End, even
    in case of errors.
val decoder_encoding : decoder -> decoder_encodingdecoder_encoding d is d's the decoded encoding scheme of d.
    Warning. If the decoder guesses the encoding or uses `UTF_16,
    rely on this value only after the first `Uchar was decoded.
val decoder_line : decoder -> intdecoder_line d is the line number of the last
    decoded (or malformed) character. See Uutf.decoder for details.val decoder_col : decoder -> intdecoder_col d is the column number of the last decoded
    (or malformed) character. See Uutf.decoder for details.val decoder_byte_count : decoder -> intdecoder_byte_count d is the number of bytes already decoded on
    d (including malformed ones). This is the last Uutf.decode's
    end byte offset counting from the beginning of the stream.val decoder_count : decoder -> intdecoder_count d is the number of characters already decoded on d
    (including malformed ones). See Uutf.decoder for details.val decoder_removed_bom : decoder -> booldecoder_removed_bom d is true iff an initial
    BOM was
    removed from the input stream. See Uutf.decoder for details.val decoder_src : decoder -> srcdecoder_src d is d's input source.val decoder_nln : decoder -> nln optiondecoder_nln d returns d's newline normalization (if any).val pp_decode : Format.formatter ->
       [< `Await | `End | `Malformed of string | `Uchar of uchar ] -> unitpp_decode ppf v prints an unspecified representation of v on
    ppf.typedst =[ `Buffer of Buffer.t | `Channel of Pervasives.out_channel | `Manual ]
`Manual destination the client
    must provide output storage with Uutf.Manual.dst.type 
val encoder : [< encoding ] -> [< dst ] -> encoderencoder encoding dst is an encoder for encoding that outputs
    to dst.
    Note. No initial
    BOM
    is encoded. If needed, this duty is left to the client.
val encode : encoder ->
       [< `Await | `End | `Uchar of uchar ] -> [ `Ok | `Partial ]encode e v is :
    `Partial iff e has a `Manual destination and needs more output
       storage. The client must use Uutf.Manual.dst to provide a new buffer
       and then call Uutf.encode with `Await until `Ok is returned.`Ok when the encoder is ready to encode a new `Uchar or `End
    For `Manual destination, encoding `End always returns
    `Partial, the client should continue as usual with `Await
    until `Ok is returned at which point Uutf.Manual.dst_rem e is
    guaranteed to be the size of the last provided buffer (i.e. nothing
    was written).
    Warning. The function assumes that u is a Unicode
    scalar value.
    If you are handling foreign data you can use Uutf.is_uchar to assert that.
    Raises. Invalid_argument if an `Uchar or `End is encoded
    after a `Partial encode.
val encoder_encoding : encoder -> encodingencoder_encoding e is e's encoding.val encoder_dst : encoder -> dstencoder_dst e is e's output destination.module Manual:sig..end
module String:sig..end
string values.
module Buffer:sig..end
Buffer.t values.
    The value of lines src is the list of lines in src as UTF-8
    encoded OCaml strings. Line breaks are determined according to the
    recommendation R4 for a readline function in section 5.8 of
    Unicode 6.1.0. If a decoding error occurs we silently replace the
    malformed sequence by the replacement character Uutf.u_rep and continue.
let lines ?encoding (src : [`Channel of in_channel | `String of string]) =
  let rec loop d buf acc = match Uutf.decode d with
  | `Uchar 0x000A ->
      let line = Buffer.contents buf in
      Buffer.clear buf; loop d buf (line :: acc)
  | `Uchar u -> Uutf.Buffer.add_utf_8 buf u; loop d buf acc
  | `End -> List.rev (Buffer.contents buf :: acc)
  | `Malformed _ -> Uutf.Buffer.add_utf_8 buf Uutf.u_rep; loop d buf acc
  | `Await -> assert false
  in
  let nln = `Readline 0x000A in
  loop (Uutf.decoder ~nln ?encoding src) (Buffer.create 512) []
  Using the `Manual interface, lines_fd does the same but on a Unix file
  descriptor.
let lines_fd ?encoding (fd : Unix.file_descr) =
  let rec loop fd s d buf acc = match Uutf.decode d with
  | `Uchar 0x000A ->
      let line = Buffer.contents buf in
      Buffer.clear buf; loop fd s d buf (line :: acc)
  | `Uchar u -> Uutf.Buffer.add_utf_8 buf u; loop fd s d buf acc
  | `End -> List.rev (Buffer.contents buf :: acc)
  | `Malformed _ -> Uutf.Buffer.add_utf_8 buf Uutf.u_rep; loop fd s d buf acc
  | `Await ->
      let rec unix_read fd s j l = try Unix.read fd s j l with
      | Unix.Unix_error (Unix.EINTR, _, _) -> unix_read fd s j l
      in
      let rc = unix_read fd s 0 (String.length s) in
      Uutf.Manual.src d s 0 rc; loop fd s d buf acc
  in
  let s = String.create 65536 (* UNIX_BUFFER_SIZE in 4.0.0 *) in
  let nln = `Readline 0x000A in
  loop fd s (Uutf.decoder ~nln ?encoding `Manual) (Buffer.create 512) []
    The result of recode src out_encoding dst has the characters of
    src written on dst with encoding out_encoding.  If a
    decoding error occurs we silently replace the malformed sequence
    by the replacement character Uutf.u_rep and continue.  Note that we
    don't add an initial
    BOM to dst,
    recoding will thus loose the initial BOM src may have. Whether
    this is a problem or not depends on the context.
let recode ?nln ?encoding out_encoding
    (src : [`Channel of in_channel | `String of string])
    (dst : [`Channel of out_channel | `Buffer of Buffer.t])
  =
  let rec loop d e = match Uutf.decode d with
  | `Uchar _ as u -> ignore (Uutf.encode e u); loop d e
  | `End -> ignore (Uutf.encode e `End)
  | `Malformed _ -> ignore (Uutf.encode e (`Uchar Uutf.u_rep)); loop d e
  | `Await -> assert false
  in
  let d = Uutf.decoder ?nln ?encoding src in
  let e = Uutf.encoder out_encoding dst in
  loop d e`Manual interface, recode_fd does the same but between
  Unix file descriptors.
let recode_fd ?nln ?encoding out_encoding
    (fdi : Unix.file_descr)
    (fdo : Unix.file_descr)
  =
  let rec encode fd s e v = match Uutf.encode e v with `Ok -> ()
  | `Partial ->
      let rec unix_write fd s j l =
        let rec write fd s j l = try Unix.single_write fd s j l with
        | Unix.Unix_error (Unix.EINTR, _, _) -> write fd s j l
        in
        let wc = write fd s j l in
        if wc < l then unix_write fd s (j + wc) (l - wc) else ()
      in
      unix_write fd s 0 (String.length s - Uutf.Manual.dst_rem e);
      Uutf.Manual.dst e s 0 (String.length s);
      encode fd s e `Await
  in
  let rec loop fdi fdo ds es d e = match Uutf.decode d with
  | `Uchar _ as u -> encode fdo es e u; loop fdi fdo ds es d e
  | `End -> encode fdo es e `End
  | `Malformed _ -> encode fdo es e (`Uchar Uutf.u_rep); loop fdi fdo ds es d e
  | `Await ->
      let rec unix_read fd s j l = try Unix.read fd s j l with
      | Unix.Unix_error (Unix.EINTR, _, _) -> unix_read fd s j l
      in
      let rc = unix_read fdi ds 0 (String.length ds) in
      Uutf.Manual.src d ds 0 rc; loop fdi fdo ds es d e
  in
  let ds = String.create 65536 (* UNIX_BUFFER_SIZE in 4.0.0 *) in
  let es = String.create 65536 (* UNIX_BUFFER_SIZE in 4.0.0 *) in
  let d = Uutf.decoder ?nln ?encoding `Manual in
  let e = Uutf.encoder out_encoding `Manual in
  Uutf.Manual.dst e es 0 (String.length es);
  loop fdi fdo ds es d e