utf32.js 9.3 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314
  1. "use strict"
  2. var Buffer = require("safer-buffer").Buffer
  3. // == UTF32-LE/BE codec. ==========================================================
  4. exports._utf32 = Utf32Codec
  5. function Utf32Codec (codecOptions, iconv) {
  6. this.iconv = iconv
  7. this.bomAware = true
  8. this.isLE = codecOptions.isLE
  9. }
  10. exports.utf32le = { type: "_utf32", isLE: true }
  11. exports.utf32be = { type: "_utf32", isLE: false }
  12. // Aliases
  13. exports.ucs4le = "utf32le"
  14. exports.ucs4be = "utf32be"
  15. Utf32Codec.prototype.encoder = Utf32Encoder
  16. Utf32Codec.prototype.decoder = Utf32Decoder
  17. // -- Encoding
  18. function Utf32Encoder (options, codec) {
  19. this.isLE = codec.isLE
  20. this.highSurrogate = 0
  21. }
  22. Utf32Encoder.prototype.write = function (str) {
  23. var src = Buffer.from(str, "ucs2")
  24. // src.length * 2 covers this chunk's code units (4 bytes each); the extra 4 bytes leave room for a
  25. // high surrogate held over from a previous chunk, which is flushed ahead of this chunk's units.
  26. var dst = Buffer.alloc(src.length * 2 + 4)
  27. var write32 = this.isLE ? dst.writeUInt32LE : dst.writeUInt32BE
  28. var offset = 0
  29. for (var i = 0; i < src.length; i += 2) {
  30. var code = src.readUInt16LE(i)
  31. var isHighSurrogate = (code >= 0xD800 && code < 0xDC00)
  32. var isLowSurrogate = (code >= 0xDC00 && code < 0xE000)
  33. if (this.highSurrogate) {
  34. if (isHighSurrogate || !isLowSurrogate) {
  35. // There shouldn't be two high surrogates in a row, nor a high surrogate which isn't followed by a low
  36. // surrogate. If this happens, keep the pending high surrogate as a stand-alone semi-invalid character
  37. // (technically wrong, but expected by some applications, like Windows file names).
  38. write32.call(dst, this.highSurrogate, offset)
  39. offset += 4
  40. } else {
  41. // Create 32-bit value from high and low surrogates;
  42. var codepoint = (((this.highSurrogate - 0xD800) << 10) | (code - 0xDC00)) + 0x10000
  43. write32.call(dst, codepoint, offset)
  44. offset += 4
  45. this.highSurrogate = 0
  46. continue
  47. }
  48. }
  49. if (isHighSurrogate) { this.highSurrogate = code } else {
  50. // Even if the current character is a low surrogate, with no previous high surrogate, we'll
  51. // encode it as a semi-invalid stand-alone character for the same reasons expressed above for
  52. // unpaired high surrogates.
  53. write32.call(dst, code, offset)
  54. offset += 4
  55. this.highSurrogate = 0
  56. }
  57. }
  58. if (offset < dst.length) { dst = dst.slice(0, offset) }
  59. return dst
  60. }
  61. Utf32Encoder.prototype.end = function () {
  62. // Treat any leftover high surrogate as a semi-valid independent character.
  63. if (!this.highSurrogate) { return }
  64. var buf = Buffer.alloc(4)
  65. if (this.isLE) { buf.writeUInt32LE(this.highSurrogate, 0) } else { buf.writeUInt32BE(this.highSurrogate, 0) }
  66. this.highSurrogate = 0
  67. return buf
  68. }
  69. // -- Decoding
  70. function Utf32Decoder (options, codec) {
  71. this.isLE = codec.isLE
  72. this.badChar = codec.iconv.defaultCharUnicode.charCodeAt(0)
  73. this.overflow = []
  74. }
  75. Utf32Decoder.prototype.write = function (src) {
  76. if (src.length === 0) { return "" }
  77. var i = 0
  78. var codepoint = 0
  79. var dst = Buffer.alloc(src.length + 4)
  80. var offset = 0
  81. var isLE = this.isLE
  82. var overflow = this.overflow
  83. var badChar = this.badChar
  84. if (overflow.length > 0) {
  85. for (; i < src.length && overflow.length < 4; i++) { overflow.push(src[i]) }
  86. if (overflow.length === 4) {
  87. // NOTE: codepoint is a signed int32 and can be negative.
  88. // NOTE: We copied this block from below to help V8 optimize it (it works with array, not buffer).
  89. if (isLE) {
  90. codepoint = overflow[0] | (overflow[1] << 8) | (overflow[2] << 16) | (overflow[3] << 24)
  91. } else {
  92. codepoint = overflow[3] | (overflow[2] << 8) | (overflow[1] << 16) | (overflow[0] << 24)
  93. }
  94. overflow.length = 0
  95. offset = _writeCodepoint(dst, offset, codepoint, badChar)
  96. }
  97. }
  98. // Main loop. Should be as optimized as possible.
  99. for (; i < src.length - 3; i += 4) {
  100. // NOTE: codepoint is a signed int32 and can be negative.
  101. if (isLE) {
  102. codepoint = src[i] | (src[i + 1] << 8) | (src[i + 2] << 16) | (src[i + 3] << 24)
  103. } else {
  104. codepoint = src[i + 3] | (src[i + 2] << 8) | (src[i + 1] << 16) | (src[i] << 24)
  105. }
  106. offset = _writeCodepoint(dst, offset, codepoint, badChar)
  107. }
  108. // Keep overflowing bytes.
  109. for (; i < src.length; i++) {
  110. overflow.push(src[i])
  111. }
  112. return dst.slice(0, offset).toString("ucs2")
  113. }
  114. function _writeCodepoint (dst, offset, codepoint, badChar) {
  115. // NOTE: codepoint is signed int32 and can be negative. We keep it that way to help V8 with optimizations.
  116. if (codepoint < 0 || codepoint > 0x10FFFF) {
  117. // Not a valid Unicode codepoint
  118. codepoint = badChar
  119. }
  120. // Ephemeral Planes: Write high surrogate.
  121. if (codepoint >= 0x10000) {
  122. codepoint -= 0x10000
  123. var high = 0xD800 | (codepoint >> 10)
  124. dst[offset++] = high & 0xff
  125. dst[offset++] = high >> 8
  126. // Low surrogate is written below.
  127. var codepoint = 0xDC00 | (codepoint & 0x3FF)
  128. }
  129. // Write BMP char or low surrogate.
  130. dst[offset++] = codepoint & 0xff
  131. dst[offset++] = codepoint >> 8
  132. return offset
  133. };
  134. Utf32Decoder.prototype.end = function () {
  135. if (this.overflow.length === 0) { return }
  136. // A leftover, incomplete 4-byte code unit at the end of the input is ill-formed. Substitute a
  137. // single U+FFFD (Unicode Standard conformance clause C10) instead of silently dropping the bytes.
  138. this.overflow.length = 0
  139. return String.fromCharCode(this.badChar)
  140. }
  141. // == UTF-32 Auto codec =============================================================
  142. // Decoder chooses automatically from UTF-32LE and UTF-32BE using BOM and space-based heuristic.
  143. // Defaults to UTF-32LE. http://en.wikipedia.org/wiki/UTF-32
  144. // Encoder/decoder default can be changed: iconv.decode(buf, 'utf32', {defaultEncoding: 'utf-32be'});
  145. // Encoder prepends BOM (which can be overridden with (addBOM: false}).
  146. exports.utf32 = Utf32AutoCodec
  147. exports.ucs4 = "utf32"
  148. function Utf32AutoCodec (options, iconv) {
  149. this.iconv = iconv
  150. }
  151. Utf32AutoCodec.prototype.encoder = Utf32AutoEncoder
  152. Utf32AutoCodec.prototype.decoder = Utf32AutoDecoder
  153. // -- Encoding
  154. function Utf32AutoEncoder (options, codec) {
  155. options = options || {}
  156. if (options.addBOM === undefined) {
  157. options.addBOM = true
  158. }
  159. this.encoder = codec.iconv.getEncoder(options.defaultEncoding || "utf-32le", options)
  160. }
  161. Utf32AutoEncoder.prototype.write = function (str) {
  162. return this.encoder.write(str)
  163. }
  164. Utf32AutoEncoder.prototype.end = function () {
  165. return this.encoder.end()
  166. }
  167. // -- Decoding
  168. function Utf32AutoDecoder (options, codec) {
  169. this.decoder = null
  170. this.initialBufs = []
  171. this.initialBufsLen = 0
  172. this.options = options || {}
  173. this.iconv = codec.iconv
  174. }
  175. Utf32AutoDecoder.prototype.write = function (buf) {
  176. if (!this.decoder) {
  177. // Codec is not chosen yet. Accumulate initial bytes.
  178. this.initialBufs.push(buf)
  179. this.initialBufsLen += buf.length
  180. if (this.initialBufsLen < 32) // We need more bytes to use space heuristic (see below)
  181. { return "" }
  182. // We have enough bytes -> detect endianness.
  183. var encoding = detectEncoding(this.initialBufs, this.options.defaultEncoding)
  184. this.decoder = this.iconv.getDecoder(encoding, this.options)
  185. var resStr = ""
  186. for (var i = 0; i < this.initialBufs.length; i++) { resStr += this.decoder.write(this.initialBufs[i]) }
  187. this.initialBufs.length = this.initialBufsLen = 0
  188. return resStr
  189. }
  190. return this.decoder.write(buf)
  191. }
  192. Utf32AutoDecoder.prototype.end = function () {
  193. if (!this.decoder) {
  194. var encoding = detectEncoding(this.initialBufs, this.options.defaultEncoding)
  195. this.decoder = this.iconv.getDecoder(encoding, this.options)
  196. var resStr = ""
  197. for (var i = 0; i < this.initialBufs.length; i++) { resStr += this.decoder.write(this.initialBufs[i]) }
  198. var trail = this.decoder.end()
  199. if (trail) { resStr += trail }
  200. this.initialBufs.length = this.initialBufsLen = 0
  201. return resStr
  202. }
  203. return this.decoder.end()
  204. }
  205. function detectEncoding (bufs, defaultEncoding) {
  206. var b = []
  207. var charsProcessed = 0
  208. var invalidLE = 0; var invalidBE = 0 // Number of invalid chars when decoded as LE or BE.
  209. var bmpCharsLE = 0; var bmpCharsBE = 0 // Number of BMP chars when decoded as LE or BE.
  210. outerLoop:
  211. for (var i = 0; i < bufs.length; i++) {
  212. var buf = bufs[i]
  213. for (var j = 0; j < buf.length; j++) {
  214. b.push(buf[j])
  215. if (b.length === 4) {
  216. if (charsProcessed === 0) {
  217. // Check BOM first.
  218. if (b[0] === 0xFF && b[1] === 0xFE && b[2] === 0 && b[3] === 0) {
  219. return "utf-32le"
  220. }
  221. if (b[0] === 0 && b[1] === 0 && b[2] === 0xFE && b[3] === 0xFF) {
  222. return "utf-32be"
  223. }
  224. }
  225. if (b[0] !== 0 || b[1] > 0x10) invalidBE++
  226. if (b[3] !== 0 || b[2] > 0x10) invalidLE++
  227. if (b[0] === 0 && b[1] === 0 && (b[2] !== 0 || b[3] !== 0)) bmpCharsBE++
  228. if ((b[0] !== 0 || b[1] !== 0) && b[2] === 0 && b[3] === 0) bmpCharsLE++
  229. b.length = 0
  230. charsProcessed++
  231. if (charsProcessed >= 100) {
  232. break outerLoop
  233. }
  234. }
  235. }
  236. }
  237. // Make decisions.
  238. if (bmpCharsBE - invalidBE > bmpCharsLE - invalidLE) return "utf-32be"
  239. if (bmpCharsBE - invalidBE < bmpCharsLE - invalidLE) return "utf-32le"
  240. // Couldn't decide (likely all zeros or not enough data).
  241. return defaultEncoding || "utf-32le"
  242. }