reader.h

Go to the documentation of this file.

#ifndef RAPIDJSON_READER_H_
#define RAPIDJSON_READER_H_

// Copyright (c) 2011 Milo Yip (miloyip@gmail.com)
// Version 0.1

#include "rapidjson.h"
#include "internal/pow10.h"
#include "internal/stack.h"
#include <csetjmp>

#ifdef RAPIDJSON_SSE42
#include <nmmintrin.h>
#elif defined(RAPIDJSON_SSE2)
#include <emmintrin.h>
#endif

#ifdef _MSC_VER
#pragma warning(push)
#pragma warning(disable : 4127) // conditional expression is constant
#endif

#ifndef RAPIDJSON_PARSE_ERROR
#define RAPIDJSON_PARSE_ERROR(msg, offset) \
        RAPIDJSON_MULTILINEMACRO_BEGIN \
        parseError_ = msg; \
        errorOffset_ = offset; \
        longjmp(jmpbuf_, 1); \
        RAPIDJSON_MULTILINEMACRO_END
#endif

namespace rapidjson {

// ParseFlag

enum ParseFlag {
        kParseDefaultFlags = 0,                 
        kParseInsituFlag = 1                    
};

// Handler

// BaseReaderHandler


template<typename Encoding = UTF8<> >
struct BaseReaderHandler {
        typedef typename Encoding::Ch Ch;

        void Default() {}
        void Null() { Default(); }
        void Bool(bool) { Default(); }
        void Int(int) { Default(); }
        void Uint(unsigned) { Default(); }
        void Int64(int64_t) { Default(); }
        void Uint64(uint64_t) { Default(); }
        void Double(double) { Default(); }
        void String(const Ch*, SizeType, bool) { Default(); }
        void StartObject() { Default(); }
        void EndObject(SizeType) { Default(); }
        void StartArray() { Default(); }
        void EndArray(SizeType) { Default(); }
};

// SkipWhitespace


template<typename Stream>
void SkipWhitespace(Stream& stream) {
        Stream s = stream;      // Use a local copy for optimization
        while (s.Peek() == ' ' || s.Peek() == '\n' || s.Peek() == '\r' || s.Peek() == '\t')
                s.Take();
        stream = s;
}

#ifdef RAPIDJSON_SSE42
inline const char *SkipWhitespace_SIMD(const char* p) {
        static const char whitespace[16] = " \n\r\t";
        __m128i w = _mm_loadu_si128((const __m128i *)&whitespace[0]);

        for (;;) {
                __m128i s = _mm_loadu_si128((const __m128i *)p);
                unsigned r = _mm_cvtsi128_si32(_mm_cmpistrm(w, s, _SIDD_UBYTE_OPS | _SIDD_CMP_EQUAL_ANY | _SIDD_BIT_MASK | _SIDD_NEGATIVE_POLARITY));
                if (r == 0)     // all 16 characters are whitespace
                        p += 16;
                else {          // some of characters may be non-whitespace
#ifdef _MSC_VER         // Find the index of first non-whitespace
                        unsigned long offset;
                        if (_BitScanForward(&offset, r))
                                return p + offset;
#else
                        if (r != 0)
                                return p + __builtin_ffs(r) - 1;
#endif
                }
        }
}

#elif defined(RAPIDJSON_SSE2)

inline const char *SkipWhitespace_SIMD(const char* p) {
        static const char whitespaces[4][17] = {
                "                ",
                "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
                "\r\r\r\r\r\r\r\r\r\r\r\r\r\r\r\r",
                "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t"};

        __m128i w0 = _mm_loadu_si128((const __m128i *)&whitespaces[0][0]);
        __m128i w1 = _mm_loadu_si128((const __m128i *)&whitespaces[1][0]);
        __m128i w2 = _mm_loadu_si128((const __m128i *)&whitespaces[2][0]);
        __m128i w3 = _mm_loadu_si128((const __m128i *)&whitespaces[3][0]);

        for (;;) {
                __m128i s = _mm_loadu_si128((const __m128i *)p);
                __m128i x = _mm_cmpeq_epi8(s, w0);
                x = _mm_or_si128(x, _mm_cmpeq_epi8(s, w1));
                x = _mm_or_si128(x, _mm_cmpeq_epi8(s, w2));
                x = _mm_or_si128(x, _mm_cmpeq_epi8(s, w3));
                unsigned short r = ~_mm_movemask_epi8(x);
                if (r == 0)     // all 16 characters are whitespace
                        p += 16;
                else {          // some of characters may be non-whitespace
#ifdef _MSC_VER         // Find the index of first non-whitespace
                        unsigned long offset;
                        if (_BitScanForward(&offset, r))
                                return p + offset;
#else
                        if (r != 0)
                                return p + __builtin_ffs(r) - 1;
#endif
                }
        }
}

#endif // RAPIDJSON_SSE2

#ifdef RAPIDJSON_SIMD
template<> inline void SkipWhitespace(InsituStringStream& stream) { 
        stream.src_ = const_cast<char*>(SkipWhitespace_SIMD(stream.src_));
}

template<> inline void SkipWhitespace(StringStream& stream) {
        stream.src_ = SkipWhitespace_SIMD(stream.src_);
}
#endif // RAPIDJSON_SIMD

// GenericReader


template <typename Encoding, typename Allocator = MemoryPoolAllocator<> >
class GenericReader {
public:
        typedef typename Encoding::Ch Ch;


        GenericReader(Allocator* allocator = 0, size_t stackCapacity = kDefaultStackCapacity) : stack_(allocator, stackCapacity), parseError_(0), errorOffset_(0) {}


        template <unsigned parseFlags, typename Stream, typename Handler>
        bool Parse(Stream& stream, Handler& handler) {
                parseError_ = 0;
                errorOffset_ = 0;

#ifdef _MSC_VER
#pragma warning(push)
#pragma warning(disable : 4611) // interaction between '_setjmp' and C++ object destruction is non-portable
#endif
                if (setjmp(jmpbuf_)) {
#ifdef _MSC_VER
#pragma warning(pop)
#endif
                        stack_.Clear();
                        return false;
                }

                SkipWhitespace(stream);

                if (stream.Peek() == '\0')
                        RAPIDJSON_PARSE_ERROR("Text only contains white space(s)", stream.Tell());
                else {
                        switch (stream.Peek()) {
                                case '{': ParseObject<parseFlags>(stream, handler); break;
                                case '[': ParseArray<parseFlags>(stream, handler); break;
                                default: RAPIDJSON_PARSE_ERROR("Expect either an object or array at root", stream.Tell());
                        }
                        SkipWhitespace(stream);

                        if (stream.Peek() != '\0')
                                RAPIDJSON_PARSE_ERROR("Nothing should follow the root object or array.", stream.Tell());
                }

                return true;
        }

        bool HasParseError() const { return parseError_ != 0; }
        const char* GetParseError() const { return parseError_; }
        size_t GetErrorOffset() const { return errorOffset_; }

private:
        // Parse object: { string : value, ... }
        template<unsigned parseFlags, typename Stream, typename Handler>
        void ParseObject(Stream& stream, Handler& handler) {
                RAPIDJSON_ASSERT(stream.Peek() == '{');
                stream.Take();  // Skip '{'
                handler.StartObject();
                SkipWhitespace(stream);

                if (stream.Peek() == '}') {
                        stream.Take();
                        handler.EndObject(0);   // empty object
                        return;
                }

                for (SizeType memberCount = 0;;) {
                        if (stream.Peek() != '"') {
                                RAPIDJSON_PARSE_ERROR("Name of an object member must be a string", stream.Tell());
                                break;
                        }

                        ParseString<parseFlags>(stream, handler);
                        SkipWhitespace(stream);

                        if (stream.Take() != ':') {
                                RAPIDJSON_PARSE_ERROR("There must be a colon after the name of object member", stream.Tell());
                                break;
                        }
                        SkipWhitespace(stream);

                        ParseValue<parseFlags>(stream, handler);
                        SkipWhitespace(stream);

                        ++memberCount;

                        switch(stream.Take()) {
                                case ',': SkipWhitespace(stream); break;
                                case '}': handler.EndObject(memberCount); return;
                                default:  RAPIDJSON_PARSE_ERROR("Must be a comma or '}' after an object member", stream.Tell());
                        }
                }
        }

        // Parse array: [ value, ... ]
        template<unsigned parseFlags, typename Stream, typename Handler>
        void ParseArray(Stream& stream, Handler& handler) {
                RAPIDJSON_ASSERT(stream.Peek() == '[');
                stream.Take();  // Skip '['
                handler.StartArray();
                SkipWhitespace(stream);

                if (stream.Peek() == ']') {
                        stream.Take();
                        handler.EndArray(0); // empty array
                        return;
                }

                for (SizeType elementCount = 0;;) {
                        ParseValue<parseFlags>(stream, handler);
                        ++elementCount;
                        SkipWhitespace(stream);

                        switch (stream.Take()) {
                                case ',': SkipWhitespace(stream); break;
                                case ']': handler.EndArray(elementCount); return;
                                default:  RAPIDJSON_PARSE_ERROR("Must be a comma or ']' after an array element.", stream.Tell());
                        }
                }
        }

        template<unsigned parseFlags, typename Stream, typename Handler>
        void ParseNull(Stream& stream, Handler& handler) {
                RAPIDJSON_ASSERT(stream.Peek() == 'n');
                stream.Take();

                if (stream.Take() == 'u' && stream.Take() == 'l' && stream.Take() == 'l')
                        handler.Null();
                else
                        RAPIDJSON_PARSE_ERROR("Invalid value", stream.Tell() - 1);
        }

        template<unsigned parseFlags, typename Stream, typename Handler>
        void ParseTrue(Stream& stream, Handler& handler) {
                RAPIDJSON_ASSERT(stream.Peek() == 't');
                stream.Take();

                if (stream.Take() == 'r' && stream.Take() == 'u' && stream.Take() == 'e')
                        handler.Bool(true);
                else
                        RAPIDJSON_PARSE_ERROR("Invalid value", stream.Tell());
        }

        template<unsigned parseFlags, typename Stream, typename Handler>
        void ParseFalse(Stream& stream, Handler& handler) {
                RAPIDJSON_ASSERT(stream.Peek() == 'f');
                stream.Take();

                if (stream.Take() == 'a' && stream.Take() == 'l' && stream.Take() == 's' && stream.Take() == 'e')
                        handler.Bool(false);
                else
                        RAPIDJSON_PARSE_ERROR("Invalid value", stream.Tell() - 1);
        }

        // Helper function to parse four hexidecimal digits in \uXXXX in ParseString().
        template<typename Stream>
        unsigned ParseHex4(Stream& stream) {
                Stream s = stream;      // Use a local copy for optimization
                unsigned codepoint = 0;
                for (int i = 0; i < 4; i++) {
                        Ch c = s.Take();
                        codepoint <<= 4;
                        codepoint += c;
                        if (c >= '0' && c <= '9')
                                codepoint -= '0';
                        else if (c >= 'A' && c <= 'F')
                                codepoint -= 'A' - 10;
                        else if (c >= 'a' && c <= 'f')
                                codepoint -= 'a' - 10;
                        else 
                                RAPIDJSON_PARSE_ERROR("Incorrect hex digit after \\u escape", s.Tell() - 1);
                }
                stream = s; // Restore stream
                return codepoint;
        }

        // Parse string, handling the prefix and suffix double quotes and escaping.
        template<unsigned parseFlags, typename Stream, typename Handler>
        void ParseString(Stream& stream, Handler& handler) {
#define Z16 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
                static const Ch escape[256] = {
                        Z16, Z16, 0, 0,'\"', 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,'/', 
                        Z16, Z16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,'\\', 0, 0, 0, 
                        0, 0,'\b', 0, 0, 0,'\f', 0, 0, 0, 0, 0, 0, 0,'\n', 0, 
                        0, 0,'\r', 0,'\t', 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
                        Z16, Z16, Z16, Z16, Z16, Z16, Z16, Z16
                };
#undef Z16

                Stream s = stream;      // Use a local copy for optimization
                RAPIDJSON_ASSERT(s.Peek() == '\"');
                s.Take();       // Skip '\"'
                Ch *head;
                SizeType len;
                if (parseFlags & kParseInsituFlag)
                        head = s.PutBegin();
                else
                        len = 0;

#define RAPIDJSON_PUT(x) \
        do { \
                if (parseFlags & kParseInsituFlag) \
                        s.Put(x); \
                else { \
                        *stack_.template Push<Ch>() = x; \
                        ++len; \
                } \
        } while(false)

                for (;;) {
                        Ch c = s.Take();
                        if (c == '\\') {        // Escape
                                Ch e = s.Take();
                                if ((sizeof(Ch) == 1 || e < 256) && escape[(unsigned char)e])
                                        RAPIDJSON_PUT(escape[(unsigned char)e]);
                                else if (e == 'u') {    // Unicode
                                        unsigned codepoint = ParseHex4(s);
                                        if (codepoint >= 0xD800 && codepoint <= 0xDBFF) { // Handle UTF-16 surrogate pair
                                                if (s.Take() != '\\' || s.Take() != 'u') {
                                                        RAPIDJSON_PARSE_ERROR("Missing the second \\u in surrogate pair", s.Tell() - 2);
                                                        return;
                                                }
                                                unsigned codepoint2 = ParseHex4(s);
                                                if (codepoint2 < 0xDC00 || codepoint2 > 0xDFFF) {
                                                        RAPIDJSON_PARSE_ERROR("The second \\u in surrogate pair is invalid", s.Tell() - 2);
                                                        return;
                                                }
                                                codepoint = (((codepoint - 0xD800) << 10) | (codepoint2 - 0xDC00)) + 0x10000;
                                        }

                                        Ch buffer[4];
                                        SizeType count = SizeType(Encoding::Encode(buffer, codepoint) - &buffer[0]);

                                        if (parseFlags & kParseInsituFlag) 
                                                for (SizeType i = 0; i < count; i++)
                                                        s.Put(buffer[i]);
                                        else {
                                                memcpy(stack_.template Push<Ch>(count), buffer, count * sizeof(Ch));
                                                len += count;
                                        }
                                }
                                else {
                                        RAPIDJSON_PARSE_ERROR("Unknown escape character", stream.Tell() - 1);
                                        return;
                                }
                        }
                        else if (c == '"') {    // Closing double quote
                                if (parseFlags & kParseInsituFlag) {
                                        size_t length = s.PutEnd(head);
                                        RAPIDJSON_ASSERT(length <= 0xFFFFFFFF);
                                        RAPIDJSON_PUT('\0');    // null-terminate the string
                                        handler.String(head, SizeType(length), false);
                                }
                                else {
                                        RAPIDJSON_PUT('\0');
                                        handler.String(stack_.template Pop<Ch>(len), len - 1, true);
                                }
                                stream = s;     // restore stream
                                return;
                        }
                        else if (c == '\0') {
                                RAPIDJSON_PARSE_ERROR("lacks ending quotation before the end of string", stream.Tell() - 1);
                                return;
                        }
                        else if ((unsigned)c < 0x20) {  // RFC 4627: unescaped = %x20-21 / %x23-5B / %x5D-10FFFF
                                RAPIDJSON_PARSE_ERROR("Incorrect unescaped character in string", stream.Tell() - 1);
                                return;
                        }
                        else
                                RAPIDJSON_PUT(c);       // Normal character, just copy
                }
#undef RAPIDJSON_PUT
        }

        template<unsigned parseFlags, typename Stream, typename Handler>
        void ParseNumber(Stream& stream, Handler& handler) {
                Stream s = stream; // Local copy for optimization
                // Parse minus
                bool minus = false;
                if (s.Peek() == '-') {
                        minus = true;
                        s.Take();
                }

                // Parse int: zero / ( digit1-9 *DIGIT )
                unsigned i;
                bool try64bit = false;
                if (s.Peek() == '0') {
                        i = 0;
                        s.Take();
                }
                else if (s.Peek() >= '1' && s.Peek() <= '9') {
                        i = s.Take() - '0';

                        if (minus)
                                while (s.Peek() >= '0' && s.Peek() <= '9') {
                                        if (i >= 214748364) { // 2^31 = 2147483648
                                                if (i != 214748364 || s.Peek() > '8') {
                                                        try64bit = true;
                                                        break;
                                                }
                                        }
                                        i = i * 10 + (s.Take() - '0');
                                }
                        else
                                while (s.Peek() >= '0' && s.Peek() <= '9') {
                                        if (i >= 429496729) { // 2^32 - 1 = 4294967295
                                                if (i != 429496729 || s.Peek() > '5') {
                                                        try64bit = true;
                                                        break;
                                                }
                                        }
                                        i = i * 10 + (s.Take() - '0');
                                }
                }
                else {
                        RAPIDJSON_PARSE_ERROR("Expect a value here.", stream.Tell());
                        return;
                }

                // Parse 64bit int
                uint64_t i64 = 0;
                bool useDouble = false;
                if (try64bit) {
                        i64 = i;
                        if (minus) 
                                while (s.Peek() >= '0' && s.Peek() <= '9') {                                    
                                        if (i64 >= 922337203685477580uLL) // 2^63 = 9223372036854775808
                                                if (i64 != 922337203685477580uLL || s.Peek() > '8') {
                                                        useDouble = true;
                                                        break;
                                                }
                                        i64 = i64 * 10 + (s.Take() - '0');
                                }
                        else
                                while (s.Peek() >= '0' && s.Peek() <= '9') {                                    
                                        if (i64 >= 1844674407370955161uLL) // 2^64 - 1 = 18446744073709551615
                                                if (i64 != 1844674407370955161uLL || s.Peek() > '5') {
                                                        useDouble = true;
                                                        break;
                                                }
                                        i64 = i64 * 10 + (s.Take() - '0');
                                }
                }

                // Force double for big integer
                double d = 0.0;
                if (useDouble) {
                        d = (double)i64;
                        while (s.Peek() >= '0' && s.Peek() <= '9') {
                                if (d >= 1E307) {
                                        RAPIDJSON_PARSE_ERROR("Number too big to store in double", stream.Tell());
                                        return;
                                }
                                d = d * 10 + (s.Take() - '0');
                        }
                }

                // Parse frac = decimal-point 1*DIGIT
                int expFrac = 0;
                if (s.Peek() == '.') {
                        if (!useDouble) {
                                d = try64bit ? (double)i64 : (double)i;
                                useDouble = true;
                        }
                        s.Take();

                        if (s.Peek() >= '0' && s.Peek() <= '9') {
                                d = d * 10 + (s.Take() - '0');
                                --expFrac;
                        }
                        else {
                                RAPIDJSON_PARSE_ERROR("At least one digit in fraction part", stream.Tell());
                                return;
                        }

                        while (s.Peek() >= '0' && s.Peek() <= '9') {
                                if (expFrac > -16) {
                                        d = d * 10 + (s.Peek() - '0');
                                        --expFrac;
                                }
                                s.Take();
                        }
                }

                // Parse exp = e [ minus / plus ] 1*DIGIT
                int exp = 0;
                if (s.Peek() == 'e' || s.Peek() == 'E') {
                        if (!useDouble) {
                                d = try64bit ? (double)i64 : (double)i;
                                useDouble = true;
                        }
                        s.Take();

                        bool expMinus = false;
                        if (s.Peek() == '+')
                                s.Take();
                        else if (s.Peek() == '-') {
                                s.Take();
                                expMinus = true;
                        }

                        if (s.Peek() >= '0' && s.Peek() <= '9') {
                                exp = s.Take() - '0';
                                while (s.Peek() >= '0' && s.Peek() <= '9') {
                                        exp = exp * 10 + (s.Take() - '0');
                                        if (exp > 308) {
                                                RAPIDJSON_PARSE_ERROR("Number too big to store in double", stream.Tell());
                                                return;
                                        }
                                }
                        }
                        else {
                                RAPIDJSON_PARSE_ERROR("At least one digit in exponent", s.Tell());
                                return;
                        }

                        if (expMinus)
                                exp = -exp;
                }

                // Finish parsing, call event according to the type of number.
                if (useDouble) {
                        d *= internal::Pow10(exp + expFrac);
                        handler.Double(minus ? -d : d);
                }
                else {
                        if (try64bit) {
                                if (minus)
                                        handler.Int64(-(int64_t)i64);
                                else
                                        handler.Uint64(i64);
                        }
                        else {
                                if (minus)
                                        handler.Int(-(int)i);
                                else
                                        handler.Uint(i);
                        }
                }

                stream = s; // restore stream
        }

        // Parse any JSON value
        template<unsigned parseFlags, typename Stream, typename Handler>
        void ParseValue(Stream& stream, Handler& handler) {
                switch (stream.Peek()) {
                        case 'n': ParseNull  <parseFlags>(stream, handler); break;
                        case 't': ParseTrue  <parseFlags>(stream, handler); break;
                        case 'f': ParseFalse <parseFlags>(stream, handler); break;
                        case '"': ParseString<parseFlags>(stream, handler); break;
                        case '{': ParseObject<parseFlags>(stream, handler); break;
                        case '[': ParseArray <parseFlags>(stream, handler); break;
                        default : ParseNumber<parseFlags>(stream, handler);
                }
        }

        static const size_t kDefaultStackCapacity = 256;        
        internal::Stack<Allocator> stack_;      
        jmp_buf jmpbuf_;                                        
        const char* parseError_;
        size_t errorOffset_;
}; // class GenericReader

typedef GenericReader<UTF8<> > Reader;

} // namespace rapidjson

#ifdef _MSC_VER
#pragma warning(pop)
#endif

#endif // RAPIDJSON_READER_H_