public static function cleanUTF8($str, $force_php = false)
{
// UTF-8 validity is checked since PHP 4.3.5
// This is an optimization: if the string is already valid UTF-8, no
// need to do PHP stuff. 99% of the time, this will be the case.
// The regexp matches the XML char production, as well as well as excluding
// non-SGML codepoints U+007F to U+009F
if (preg_match('/^[\\x{9}\\x{A}\\x{D}\\x{20}-\\x{7E}\\x{A0}-\\x{D7FF}\\x{E000}-\\x{FFFD}\\x{10000}-\\x{10FFFF}]*$/Du', $str)) {
return $str;
}
$mState = 0;
// cached expected number of octets after the current octet
// until the beginning of the next UTF8 character sequence
$mUcs4 = 0;
// cached Unicode character
$mBytes = 1;
// cached expected number of octets in the current sequence
// original code involved an $out that was an array of Unicode
// codepoints. Instead of having to convert back into UTF-8, we've
// decided to directly append valid UTF-8 characters onto a string
// $out once they're done. $char accumulates raw bytes, while $mUcs4
// turns into the Unicode code point, so there's some redundancy.
$out = '';
$char = '';
$len = strlen($str);
for ($i = 0; $i < $len; $i++) {
$in = ord($str[$i]);
$char .= $str[$i];
// append byte to char
if (0 == $mState) {
// When mState is zero we expect either a US-ASCII character
// or a multi-octet sequence.
if (0 == (0x80 & $in)) {
// US-ASCII, pass straight through.
if (($in <= 31 || $in == 127) && !($in == 9 || $in == 13 || $in == 10)) {
// control characters, remove
} else {
$out .= $char;
}
// reset
$char = '';
$mBytes = 1;
} elseif (0xc0 == (0xe0 & $in)) {
// First octet of 2 octet sequence
$mUcs4 = $in;
$mUcs4 = ($mUcs4 & 0x1f) << 6;
$mState = 1;
$mBytes = 2;
} elseif (0xe0 == (0xf0 & $in)) {
// First octet of 3 octet sequence
$mUcs4 = $in;
$mUcs4 = ($mUcs4 & 0xf) << 12;
$mState = 2;
$mBytes = 3;
} elseif (0xf0 == (0xf8 & $in)) {
// First octet of 4 octet sequence
$mUcs4 = $in;
$mUcs4 = ($mUcs4 & 0x7) << 18;
$mState = 3;
$mBytes = 4;
} elseif (0xf8 == (0xfc & $in)) {
// First octet of 5 octet sequence.
//
// This is illegal because the encoded codepoint must be
// either:
// (a) not the shortest form or
// (b) outside the Unicode range of 0-0x10FFFF.
// Rather than trying to resynchronize, we will carry on
// until the end of the sequence and let the later error
// handling code catch it.
$mUcs4 = $in;
$mUcs4 = ($mUcs4 & 0x3) << 24;
$mState = 4;
$mBytes = 5;
} elseif (0xfc == (0xfe & $in)) {
// First octet of 6 octet sequence, see comments for 5
// octet sequence.
$mUcs4 = $in;
$mUcs4 = ($mUcs4 & 1) << 30;
$mState = 5;
$mBytes = 6;
} else {
// Current octet is neither in the US-ASCII range nor a
// legal first octet of a multi-octet sequence.
$mState = 0;
$mUcs4 = 0;
$mBytes = 1;
$char = '';
}
} else {
// When mState is non-zero, we expect a continuation of the
// multi-octet sequence
if (0x80 == (0xc0 & $in)) {
// Legal continuation.
$shift = ($mState - 1) * 6;
$tmp = $in;
$tmp = ($tmp & 0x3f) << $shift;
$mUcs4 |= $tmp;
if (0 == --$mState) {
// End of the multi-octet sequence. mUcs4 now contains
// the final Unicode codepoint to be output
// Check for illegal sequences and codepoints.
// From Unicode 3.1, non-shortest form is illegal
if (2 == $mBytes && $mUcs4 < 0x80 || 3 == $mBytes && $mUcs4 < 0x800 || 4 == $mBytes && $mUcs4 < 0x10000 || 4 < $mBytes || ($mUcs4 & 0xfffff800) == 0xd800 || $mUcs4 > 0x10ffff) {
} elseif (0xfeff != $mUcs4 && (0x9 == $mUcs4 || 0xa == $mUcs4 || 0xd == $mUcs4 || 0x20 <= $mUcs4 && 0x7e >= $mUcs4 || 0xa0 <= $mUcs4 && 0xd7ff >= $mUcs4 || 0x10000 <= $mUcs4 && 0x10ffff >= $mUcs4)) {
$out .= $char;
}
// initialize UTF8 cache (reset)
$mState = 0;
$mUcs4 = 0;
$mBytes = 1;
$char = '';
}
} else {
// ((0xC0 & (*in) != 0x80) && (mState != 0))
// Incomplete multi-octet sequence.
// used to result in complete fail, but we'll reset
$mState = 0;
$mUcs4 = 0;
$mBytes = 1;
$char = '';
}
}
}
return $out;
}
