summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat (limited to '3rdParty/Boost/boost/regex/pending/unicode_iterator.hpp')
-rw-r--r--3rdParty/Boost/boost/regex/pending/unicode_iterator.hpp692
1 files changed, 692 insertions, 0 deletions
diff --git a/3rdParty/Boost/boost/regex/pending/unicode_iterator.hpp b/3rdParty/Boost/boost/regex/pending/unicode_iterator.hpp
new file mode 100644
index 0000000..657ca0a
--- /dev/null
+++ b/3rdParty/Boost/boost/regex/pending/unicode_iterator.hpp
@@ -0,0 +1,692 @@
+/*
+ *
+ * Copyright (c) 2004
+ * John Maddock
+ *
+ * Use, modification and distribution are subject to the
+ * Boost Software License, Version 1.0. (See accompanying file
+ * LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+ *
+ */
+
+ /*
+ * LOCATION: see http://www.boost.org for most recent version.
+ * FILE unicode_iterator.hpp
+ * VERSION see <boost/version.hpp>
+ * DESCRIPTION: Iterator adapters for converting between different Unicode encodings.
+ */
+
+/****************************************************************************
+
+Contents:
+~~~~~~~~~
+
+1) Read Only, Input Adapters:
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+template <class BaseIterator, class U8Type = ::boost::uint8_t>
+class u32_to_u8_iterator;
+
+Adapts sequence of UTF-32 code points to "look like" a sequence of UTF-8.
+
+template <class BaseIterator, class U32Type = ::boost::uint32_t>
+class u8_to_u32_iterator;
+
+Adapts sequence of UTF-8 code points to "look like" a sequence of UTF-32.
+
+template <class BaseIterator, class U16Type = ::boost::uint16_t>
+class u32_to_u16_iterator;
+
+Adapts sequence of UTF-32 code points to "look like" a sequence of UTF-16.
+
+template <class BaseIterator, class U32Type = ::boost::uint32_t>
+class u16_to_u32_iterator;
+
+Adapts sequence of UTF-16 code points to "look like" a sequence of UTF-32.
+
+2) Single pass output iterator adapters:
+
+template <class BaseIterator>
+class utf8_output_iterator;
+
+Accepts UTF-32 code points and forwards them on as UTF-8 code points.
+
+template <class BaseIterator>
+class utf16_output_iterator;
+
+Accepts UTF-32 code points and forwards them on as UTF-16 code points.
+
+****************************************************************************/
+
+#ifndef BOOST_REGEX_UNICODE_ITERATOR_HPP
+#define BOOST_REGEX_UNICODE_ITERATOR_HPP
+#include <boost/cstdint.hpp>
+#include <boost/assert.hpp>
+#include <boost/iterator/iterator_facade.hpp>
+#include <boost/static_assert.hpp>
+#include <boost/throw_exception.hpp>
+#include <stdexcept>
+#ifndef BOOST_NO_STD_LOCALE
+#include <sstream>
+#include <ios>
+#endif
+#include <limits.h> // CHAR_BIT
+
+namespace boost{
+
+namespace detail{
+
+static const ::boost::uint16_t high_surrogate_base = 0xD7C0u;
+static const ::boost::uint16_t low_surrogate_base = 0xDC00u;
+static const ::boost::uint32_t ten_bit_mask = 0x3FFu;
+
+inline bool is_high_surrogate(::boost::uint16_t v)
+{
+ return (v & 0xFC00u) == 0xd800u;
+}
+inline bool is_low_surrogate(::boost::uint16_t v)
+{
+ return (v & 0xFC00u) == 0xdc00u;
+}
+template <class T>
+inline bool is_surrogate(T v)
+{
+ return (v & 0xF800u) == 0xd800;
+}
+
+inline unsigned utf8_byte_count(boost::uint8_t c)
+{
+ // if the most significant bit with a zero in it is in position
+ // 8-N then there are N bytes in this UTF-8 sequence:
+ boost::uint8_t mask = 0x80u;
+ unsigned result = 0;
+ while(c & mask)
+ {
+ ++result;
+ mask >>= 1;
+ }
+ return (result == 0) ? 1 : ((result > 4) ? 4 : result);
+}
+
+inline unsigned utf8_trailing_byte_count(boost::uint8_t c)
+{
+ return utf8_byte_count(c) - 1;
+}
+
+inline void invalid_utf32_code_point(::boost::uint32_t val)
+{
+#ifndef BOOST_NO_STD_LOCALE
+ std::stringstream ss;
+ ss << "Invalid UTF-32 code point U+" << std::showbase << std::hex << val << " encountered while trying to encode UTF-16 sequence";
+ std::out_of_range e(ss.str());
+#else
+ std::out_of_range e("Invalid UTF-32 code point encountered while trying to encode UTF-16 sequence");
+#endif
+ boost::throw_exception(e);
+}
+
+
+} // namespace detail
+
+template <class BaseIterator, class U16Type = ::boost::uint16_t>
+class u32_to_u16_iterator
+ : public boost::iterator_facade<u32_to_u16_iterator<BaseIterator, U16Type>, U16Type, std::bidirectional_iterator_tag, const U16Type>
+{
+ typedef boost::iterator_facade<u32_to_u16_iterator<BaseIterator, U16Type>, U16Type, std::bidirectional_iterator_tag, const U16Type> base_type;
+
+#if !defined(BOOST_NO_STD_ITERATOR_TRAITS) && !defined(BOOST_NO_TEMPLATE_PARTIAL_SPECIALIZATION)
+ typedef typename std::iterator_traits<BaseIterator>::value_type base_value_type;
+
+ BOOST_STATIC_ASSERT(sizeof(base_value_type)*CHAR_BIT == 32);
+ BOOST_STATIC_ASSERT(sizeof(U16Type)*CHAR_BIT == 16);
+#endif
+
+public:
+ typename base_type::reference
+ dereference()const
+ {
+ if(m_current == 2)
+ extract_current();
+ return m_values[m_current];
+ }
+ bool equal(const u32_to_u16_iterator& that)const
+ {
+ if(m_position == that.m_position)
+ {
+ // Both m_currents must be equal, or both even
+ // this is the same as saying their sum must be even:
+ return (m_current + that.m_current) & 1u ? false : true;
+ }
+ return false;
+ }
+ void increment()
+ {
+ // if we have a pending read then read now, so that we know whether
+ // to skip a position, or move to a low-surrogate:
+ if(m_current == 2)
+ {
+ // pending read:
+ extract_current();
+ }
+ // move to the next surrogate position:
+ ++m_current;
+ // if we've reached the end skip a position:
+ if(m_values[m_current] == 0)
+ {
+ m_current = 2;
+ ++m_position;
+ }
+ }
+ void decrement()
+ {
+ if(m_current != 1)
+ {
+ // decrementing an iterator always leads to a valid position:
+ --m_position;
+ extract_current();
+ m_current = m_values[1] ? 1 : 0;
+ }
+ else
+ {
+ m_current = 0;
+ }
+ }
+ BaseIterator base()const
+ {
+ return m_position;
+ }
+ // construct:
+ u32_to_u16_iterator() : m_position(), m_current(0)
+ {
+ m_values[0] = 0;
+ m_values[1] = 0;
+ m_values[2] = 0;
+ }
+ u32_to_u16_iterator(BaseIterator b) : m_position(b), m_current(2)
+ {
+ m_values[0] = 0;
+ m_values[1] = 0;
+ m_values[2] = 0;
+ }
+private:
+
+ void extract_current()const
+ {
+ // begin by checking for a code point out of range:
+ ::boost::uint32_t v = *m_position;
+ if(v >= 0x10000u)
+ {
+ if(v > 0x10FFFFu)
+ detail::invalid_utf32_code_point(*m_position);
+ // split into two surrogates:
+ m_values[0] = static_cast<U16Type>(v >> 10) + detail::high_surrogate_base;
+ m_values[1] = static_cast<U16Type>(v & detail::ten_bit_mask) + detail::low_surrogate_base;
+ m_current = 0;
+ BOOST_ASSERT(detail::is_high_surrogate(m_values[0]));
+ BOOST_ASSERT(detail::is_low_surrogate(m_values[1]));
+ }
+ else
+ {
+ // 16-bit code point:
+ m_values[0] = static_cast<U16Type>(*m_position);
+ m_values[1] = 0;
+ m_current = 0;
+ // value must not be a surrogate:
+ if(detail::is_surrogate(m_values[0]))
+ detail::invalid_utf32_code_point(*m_position);
+ }
+ }
+ BaseIterator m_position;
+ mutable U16Type m_values[3];
+ mutable unsigned m_current;
+};
+
+template <class BaseIterator, class U32Type = ::boost::uint32_t>
+class u16_to_u32_iterator
+ : public boost::iterator_facade<u16_to_u32_iterator<BaseIterator, U32Type>, U32Type, std::bidirectional_iterator_tag, const U32Type>
+{
+ typedef boost::iterator_facade<u16_to_u32_iterator<BaseIterator, U32Type>, U32Type, std::bidirectional_iterator_tag, const U32Type> base_type;
+ // special values for pending iterator reads:
+ BOOST_STATIC_CONSTANT(U32Type, pending_read = 0xffffffffu);
+
+#if !defined(BOOST_NO_STD_ITERATOR_TRAITS) && !defined(BOOST_NO_TEMPLATE_PARTIAL_SPECIALIZATION)
+ typedef typename std::iterator_traits<BaseIterator>::value_type base_value_type;
+
+ BOOST_STATIC_ASSERT(sizeof(base_value_type)*CHAR_BIT == 16);
+ BOOST_STATIC_ASSERT(sizeof(U32Type)*CHAR_BIT == 32);
+#endif
+
+public:
+ typename base_type::reference
+ dereference()const
+ {
+ if(m_value == pending_read)
+ extract_current();
+ return m_value;
+ }
+ bool equal(const u16_to_u32_iterator& that)const
+ {
+ return m_position == that.m_position;
+ }
+ void increment()
+ {
+ // skip high surrogate first if there is one:
+ if(detail::is_high_surrogate(*m_position)) ++m_position;
+ ++m_position;
+ m_value = pending_read;
+ }
+ void decrement()
+ {
+ --m_position;
+ // if we have a low surrogate then go back one more:
+ if(detail::is_low_surrogate(*m_position))
+ --m_position;
+ m_value = pending_read;
+ }
+ BaseIterator base()const
+ {
+ return m_position;
+ }
+ // construct:
+ u16_to_u32_iterator() : m_position()
+ {
+ m_value = pending_read;
+ }
+ u16_to_u32_iterator(BaseIterator b) : m_position(b)
+ {
+ m_value = pending_read;
+ }
+private:
+ static void invalid_code_point(::boost::uint16_t val)
+ {
+#ifndef BOOST_NO_STD_LOCALE
+ std::stringstream ss;
+ ss << "Misplaced UTF-16 surrogate U+" << std::showbase << std::hex << val << " encountered while trying to encode UTF-32 sequence";
+ std::out_of_range e(ss.str());
+#else
+ std::out_of_range e("Misplaced UTF-16 surrogate encountered while trying to encode UTF-32 sequence");
+#endif
+ boost::throw_exception(e);
+ }
+ void extract_current()const
+ {
+ m_value = static_cast<U32Type>(static_cast< ::boost::uint16_t>(*m_position));
+ // if the last value is a high surrogate then adjust m_position and m_value as needed:
+ if(detail::is_high_surrogate(*m_position))
+ {
+ // precondition; next value must have be a low-surrogate:
+ BaseIterator next(m_position);
+ ::boost::uint16_t t = *++next;
+ if((t & 0xFC00u) != 0xDC00u)
+ invalid_code_point(t);
+ m_value = (m_value - detail::high_surrogate_base) << 10;
+ m_value |= (static_cast<U32Type>(static_cast< ::boost::uint16_t>(t)) & detail::ten_bit_mask);
+ }
+ // postcondition; result must not be a surrogate:
+ if(detail::is_surrogate(m_value))
+ invalid_code_point(static_cast< ::boost::uint16_t>(m_value));
+ }
+ BaseIterator m_position;
+ mutable U32Type m_value;
+};
+
+template <class BaseIterator, class U8Type = ::boost::uint8_t>
+class u32_to_u8_iterator
+ : public boost::iterator_facade<u32_to_u8_iterator<BaseIterator, U8Type>, U8Type, std::bidirectional_iterator_tag, const U8Type>
+{
+ typedef boost::iterator_facade<u32_to_u8_iterator<BaseIterator, U8Type>, U8Type, std::bidirectional_iterator_tag, const U8Type> base_type;
+
+#if !defined(BOOST_NO_STD_ITERATOR_TRAITS) && !defined(BOOST_NO_TEMPLATE_PARTIAL_SPECIALIZATION)
+ typedef typename std::iterator_traits<BaseIterator>::value_type base_value_type;
+
+ BOOST_STATIC_ASSERT(sizeof(base_value_type)*CHAR_BIT == 32);
+ BOOST_STATIC_ASSERT(sizeof(U8Type)*CHAR_BIT == 8);
+#endif
+
+public:
+ typename base_type::reference
+ dereference()const
+ {
+ if(m_current == 4)
+ extract_current();
+ return m_values[m_current];
+ }
+ bool equal(const u32_to_u8_iterator& that)const
+ {
+ if(m_position == that.m_position)
+ {
+ // either the m_current's must be equal, or one must be 0 and
+ // the other 4: which means neither must have bits 1 or 2 set:
+ return (m_current == that.m_current)
+ || (((m_current | that.m_current) & 3) == 0);
+ }
+ return false;
+ }
+ void increment()
+ {
+ // if we have a pending read then read now, so that we know whether
+ // to skip a position, or move to a low-surrogate:
+ if(m_current == 4)
+ {
+ // pending read:
+ extract_current();
+ }
+ // move to the next surrogate position:
+ ++m_current;
+ // if we've reached the end skip a position:
+ if(m_values[m_current] == 0)
+ {
+ m_current = 4;
+ ++m_position;
+ }
+ }
+ void decrement()
+ {
+ if((m_current & 3) == 0)
+ {
+ --m_position;
+ extract_current();
+ m_current = 3;
+ while(m_current && (m_values[m_current] == 0))
+ --m_current;
+ }
+ else
+ --m_current;
+ }
+ BaseIterator base()const
+ {
+ return m_position;
+ }
+ // construct:
+ u32_to_u8_iterator() : m_position(), m_current(0)
+ {
+ m_values[0] = 0;
+ m_values[1] = 0;
+ m_values[2] = 0;
+ m_values[3] = 0;
+ m_values[4] = 0;
+ }
+ u32_to_u8_iterator(BaseIterator b) : m_position(b), m_current(4)
+ {
+ m_values[0] = 0;
+ m_values[1] = 0;
+ m_values[2] = 0;
+ m_values[3] = 0;
+ m_values[4] = 0;
+ }
+private:
+
+ void extract_current()const
+ {
+ boost::uint32_t c = *m_position;
+ if(c > 0x10FFFFu)
+ detail::invalid_utf32_code_point(c);
+ if(c < 0x80u)
+ {
+ m_values[0] = static_cast<unsigned char>(c);
+ m_values[1] = static_cast<unsigned char>(0u);
+ m_values[2] = static_cast<unsigned char>(0u);
+ m_values[3] = static_cast<unsigned char>(0u);
+ }
+ else if(c < 0x800u)
+ {
+ m_values[0] = static_cast<unsigned char>(0xC0u + (c >> 6));
+ m_values[1] = static_cast<unsigned char>(0x80u + (c & 0x3Fu));
+ m_values[2] = static_cast<unsigned char>(0u);
+ m_values[3] = static_cast<unsigned char>(0u);
+ }
+ else if(c < 0x10000u)
+ {
+ m_values[0] = static_cast<unsigned char>(0xE0u + (c >> 12));
+ m_values[1] = static_cast<unsigned char>(0x80u + ((c >> 6) & 0x3Fu));
+ m_values[2] = static_cast<unsigned char>(0x80u + (c & 0x3Fu));
+ m_values[3] = static_cast<unsigned char>(0u);
+ }
+ else
+ {
+ m_values[0] = static_cast<unsigned char>(0xF0u + (c >> 18));
+ m_values[1] = static_cast<unsigned char>(0x80u + ((c >> 12) & 0x3Fu));
+ m_values[2] = static_cast<unsigned char>(0x80u + ((c >> 6) & 0x3Fu));
+ m_values[3] = static_cast<unsigned char>(0x80u + (c & 0x3Fu));
+ }
+ m_current= 0;
+ }
+ BaseIterator m_position;
+ mutable U8Type m_values[5];
+ mutable unsigned m_current;
+};
+
+template <class BaseIterator, class U32Type = ::boost::uint32_t>
+class u8_to_u32_iterator
+ : public boost::iterator_facade<u8_to_u32_iterator<BaseIterator, U32Type>, U32Type, std::bidirectional_iterator_tag, const U32Type>
+{
+ typedef boost::iterator_facade<u8_to_u32_iterator<BaseIterator, U32Type>, U32Type, std::bidirectional_iterator_tag, const U32Type> base_type;
+ // special values for pending iterator reads:
+ BOOST_STATIC_CONSTANT(U32Type, pending_read = 0xffffffffu);
+
+#if !defined(BOOST_NO_STD_ITERATOR_TRAITS) && !defined(BOOST_NO_TEMPLATE_PARTIAL_SPECIALIZATION)
+ typedef typename std::iterator_traits<BaseIterator>::value_type base_value_type;
+
+ BOOST_STATIC_ASSERT(sizeof(base_value_type)*CHAR_BIT == 8);
+ BOOST_STATIC_ASSERT(sizeof(U32Type)*CHAR_BIT == 32);
+#endif
+
+public:
+ typename base_type::reference
+ dereference()const
+ {
+ if(m_value == pending_read)
+ extract_current();
+ return m_value;
+ }
+ bool equal(const u8_to_u32_iterator& that)const
+ {
+ return m_position == that.m_position;
+ }
+ void increment()
+ {
+ // skip high surrogate first if there is one:
+ unsigned c = detail::utf8_byte_count(*m_position);
+ std::advance(m_position, c);
+ m_value = pending_read;
+ }
+ void decrement()
+ {
+ // Keep backtracking until we don't have a trailing character:
+ unsigned count = 0;
+ while((*--m_position & 0xC0u) == 0x80u) ++count;
+ // now check that the sequence was valid:
+ if(count != detail::utf8_trailing_byte_count(*m_position))
+ invalid_sequnce();
+ m_value = pending_read;
+ }
+ BaseIterator base()const
+ {
+ return m_position;
+ }
+ // construct:
+ u8_to_u32_iterator() : m_position()
+ {
+ m_value = pending_read;
+ }
+ u8_to_u32_iterator(BaseIterator b) : m_position(b)
+ {
+ m_value = pending_read;
+ }
+private:
+ static void invalid_sequnce()
+ {
+ std::out_of_range e("Invalid UTF-8 sequence encountered while trying to encode UTF-32 character");
+ boost::throw_exception(e);
+ }
+ void extract_current()const
+ {
+ m_value = static_cast<U32Type>(static_cast< ::boost::uint8_t>(*m_position));
+ // we must not have a continuation character:
+ if((m_value & 0xC0u) == 0x80u)
+ invalid_sequnce();
+ // see how many extra byts we have:
+ unsigned extra = detail::utf8_trailing_byte_count(*m_position);
+ // extract the extra bits, 6 from each extra byte:
+ BaseIterator next(m_position);
+ for(unsigned c = 0; c < extra; ++c)
+ {
+ ++next;
+ m_value <<= 6;
+ m_value += static_cast<boost::uint8_t>(*next) & 0x3Fu;
+ }
+ // we now need to remove a few of the leftmost bits, but how many depends
+ // upon how many extra bytes we've extracted:
+ static const boost::uint32_t masks[4] =
+ {
+ 0x7Fu,
+ 0x7FFu,
+ 0xFFFFu,
+ 0x1FFFFFu,
+ };
+ m_value &= masks[extra];
+ // check the result:
+ if(m_value > static_cast<U32Type>(0x10FFFFu))
+ invalid_sequnce();
+ }
+ BaseIterator m_position;
+ mutable U32Type m_value;
+};
+
+template <class BaseIterator>
+class utf16_output_iterator
+{
+public:
+ typedef void difference_type;
+ typedef void value_type;
+ typedef boost::uint32_t* pointer;
+ typedef boost::uint32_t& reference;
+ typedef std::output_iterator_tag iterator_category;
+
+ utf16_output_iterator(const BaseIterator& b)
+ : m_position(b){}
+ utf16_output_iterator(const utf16_output_iterator& that)
+ : m_position(that.m_position){}
+ utf16_output_iterator& operator=(const utf16_output_iterator& that)
+ {
+ m_position = that.m_position;
+ return *this;
+ }
+ const utf16_output_iterator& operator*()const
+ {
+ return *this;
+ }
+ void operator=(boost::uint32_t val)const
+ {
+ push(val);
+ }
+ utf16_output_iterator& operator++()
+ {
+ return *this;
+ }
+ utf16_output_iterator& operator++(int)
+ {
+ return *this;
+ }
+ BaseIterator base()const
+ {
+ return m_position;
+ }
+private:
+ void push(boost::uint32_t v)const
+ {
+ if(v >= 0x10000u)
+ {
+ // begin by checking for a code point out of range:
+ if(v > 0x10FFFFu)
+ detail::invalid_utf32_code_point(v);
+ // split into two surrogates:
+ *m_position++ = static_cast<boost::uint16_t>(v >> 10) + detail::high_surrogate_base;
+ *m_position++ = static_cast<boost::uint16_t>(v & detail::ten_bit_mask) + detail::low_surrogate_base;
+ }
+ else
+ {
+ // 16-bit code point:
+ // value must not be a surrogate:
+ if(detail::is_surrogate(v))
+ detail::invalid_utf32_code_point(v);
+ *m_position++ = static_cast<boost::uint16_t>(v);
+ }
+ }
+ mutable BaseIterator m_position;
+};
+
+template <class BaseIterator>
+class utf8_output_iterator
+{
+public:
+ typedef void difference_type;
+ typedef void value_type;
+ typedef boost::uint32_t* pointer;
+ typedef boost::uint32_t& reference;
+ typedef std::output_iterator_tag iterator_category;
+
+ utf8_output_iterator(const BaseIterator& b)
+ : m_position(b){}
+ utf8_output_iterator(const utf8_output_iterator& that)
+ : m_position(that.m_position){}
+ utf8_output_iterator& operator=(const utf8_output_iterator& that)
+ {
+ m_position = that.m_position;
+ return *this;
+ }
+ const utf8_output_iterator& operator*()const
+ {
+ return *this;
+ }
+ void operator=(boost::uint32_t val)const
+ {
+ push(val);
+ }
+ utf8_output_iterator& operator++()
+ {
+ return *this;
+ }
+ utf8_output_iterator& operator++(int)
+ {
+ return *this;
+ }
+ BaseIterator base()const
+ {
+ return m_position;
+ }
+private:
+ void push(boost::uint32_t c)const
+ {
+ if(c > 0x10FFFFu)
+ detail::invalid_utf32_code_point(c);
+ if(c < 0x80u)
+ {
+ *m_position++ = static_cast<unsigned char>(c);
+ }
+ else if(c < 0x800u)
+ {
+ *m_position++ = static_cast<unsigned char>(0xC0u + (c >> 6));
+ *m_position++ = static_cast<unsigned char>(0x80u + (c & 0x3Fu));
+ }
+ else if(c < 0x10000u)
+ {
+ *m_position++ = static_cast<unsigned char>(0xE0u + (c >> 12));
+ *m_position++ = static_cast<unsigned char>(0x80u + ((c >> 6) & 0x3Fu));
+ *m_position++ = static_cast<unsigned char>(0x80u + (c & 0x3Fu));
+ }
+ else
+ {
+ *m_position++ = static_cast<unsigned char>(0xF0u + (c >> 18));
+ *m_position++ = static_cast<unsigned char>(0x80u + ((c >> 12) & 0x3Fu));
+ *m_position++ = static_cast<unsigned char>(0x80u + ((c >> 6) & 0x3Fu));
+ *m_position++ = static_cast<unsigned char>(0x80u + (c & 0x3Fu));
+ }
+ }
+ mutable BaseIterator m_position;
+};
+
+} // namespace boost
+
+#endif // BOOST_REGEX_UNICODE_ITERATOR_HPP
+