iconv用法解读

iconv是一个字符集转换函数，原型为： size_t iconv(iconv_t cd,              char **inbuf, size_t *inbytesleft,              char **outbuf, size_t *outbytesleft);

1. // 传递给do_convert的in_buf，所有字节数（in_buf_size指定）都是可以转换成功的
2. static int do_convert(iconv_t cd, const char* from, size_t from_size, std::string* to)
3. {
4.     char* in_buf_ptr = const_cast<char*>(from);
5.     size_t in_bytes_left = from_size;
6.     size_t out_bytes = in_bytes_left*3 + 1;
7.     size_t out_bytes_left = out_bytes;
8.     std::string out(out_bytes_left, '\0');
9.     char* out_buf_start = const_cast<char*>(out.c_str());
10.     char* out_buf_ptr = out_buf_start;
12. int bytes = iconv(cd, &in_buf_ptr, &in_bytes_left, &out_buf_ptr, &out_bytes_left);
13. if (-1 == bytes)
14.         return errno;
16. to->assign(out_buf_start, out_bytes-out_bytes_left);
17.     return 0;
18. }
20. // 可忽略不能转换的部分，
21. // 也可以在结果中保留不能被转换的部分
22. // 详细实现可以浏览：
23. // https://github.com/eyjian/mooon/blob/master/common_library/src/utils/charset_utils.cpp
24. void CCharsetUtils::convert(const std::string& from_charset, const std::string& to_charset,
25. const std::string& from, std::string* to,
26.                             bool ignore_error, bool skip_error) throw (CException)
27. {
28.     std::string result; // 用来保存处理后的内容
29.     char* in_buf = const_cast<char*>(from.c_str());
30.     size_t in_bytes = from.size(); // 需要处理的总字节数
31.     size_t in_bytes_left = in_bytes; // 剩余的未被处理的字节数
32.     iconv_t cd = iconv_open(to_charset.c_str(), from_charset.c_str());
34. if ((iconv_t)(-1) == cd)
35. {
36.         THROW_EXCEPTION(strerror(errno), errno);
37. }
38. while (in_bytes_left > 0)
39. {
40. int errcode;
41.         size_t out_bytes = in_bytes_left * 3 + 1; // 保证足够大
42.         size_t out_bytes_left = out_bytes;
43.         std::string out(out_bytes_left, '\0');
44.         char* out_buf = const_cast<char*>(out.c_str());
45.         char* out_buf_start = out_buf;
46.         char* in_buf_start = in_buf;
48. // 如果成功，返回值bytes为0
49. // 如果成功，in_buf指向in的结尾符，即'\0'，同时in_bytes_left值为0
50. // 如果失败，in_buf指向未能转换的起始地址，而in_bytes_left值为剩余的未被转换的（可能含有可转换的）字节数
51. // 如果成功，则out_bytes-out_bytes_left值为转换后的字节数
52. // 如果成功，则out_buf_start存储了被转换后的结果，有效长度为out_bytes-out_bytes_left
53. int bytes = iconv(cd, &in_buf, &in_bytes_left, &out_buf, &out_bytes_left);
54. if (bytes != -1)
55. {
56.             result.append(out_buf_start, out_bytes-out_bytes_left);
57.             break;
58. }
59. else if (!ignore_error)
60. {
61.             errcode = errno;
62.             iconv_close(cd);
63.             THROW_EXCEPTION(strerror(errcode), errcode);
64. }
65. else
66. {
67. // EILSEQ An invalid multibyte sequence has been encountered in the input.
68. // EINVAL An incomplete multibyte sequence has been encountered in the input.
69. if ((errno != EINVAL) &&
70. (errno != EILSEQ))
71. {
72. // E2BIG There is not sufficient room at *outbuf.
73.                 errcode = errno;
74.                 iconv_close(cd);
75.                 THROW_EXCEPTION(strerror(errcode), errcode);
76. }
77. else
78. {
79. // in_buf之前部分是可以转换的
80. if (in_buf != in_buf_start)
81. {
82.                     std::string str;
83.                     errcode = do_convert(cd, in_buf_start, in_buf-in_buf_start, &str);
84. if (errcode != 0)
85. {
86.                         iconv_close(cd);
87.                         THROW_EXCEPTION(strerror(errcode), errcode);
88. }
90.                     result.append(str);
91. }
93. // skip_error决定未能被转换的是否出现在结果当中
94. if (!skip_error)
95. {
96.                     result.append(in_buf, 1);
97. }
99. // 往前推进
100. --in_bytes_left; // 将导致while语句结束
101. ++in_buf;
102. }
103. }
104. }
106. if (-1 == iconv_close(cd))
107. {
108.         THROW_EXCEPTION(strerror(errno), errno);
109. }
111. // 不能直接使用to，因为to可能就是from
112. *to = result;
113. }
115. void CCharsetUtils::gbk_to_utf8(const std::string& from, std::string* to, bool ignore_error, bool skip_error) throw (CException)
116. {
117.     convert("gbk", "utf-8", from, to, ignore_error, skip_error);
118. }
120. void CCharsetUtils::utf8_to_gbk(const std::string& from, std::string* to, bool ignore_error, bool skip_error) throw (CException)
121. {
122.     convert("utf-8", "gbk", from, to, ignore_error, skip_error);
123. }
125. void CCharsetUtils::gb2312_to_utf8(const std::string& from, std::string* to, bool ignore_error, bool skip_error) throw (CException)
126. {
127.     convert("gb2312", "utf-8", from, to, ignore_error, skip_error);
128. }
130. void CCharsetUtils::utf8_to_gb2312(const std::string& from, std::string* to, bool ignore_error, bool skip_error) throw (CException)
131. {
132.     convert("utf-8", "gb2312", from, to, ignore_error, skip_error);
133. }