????
Current Path : /lib/python2.7/site-packages/google/protobuf/ |
Current File : //lib/python2.7/site-packages/google/protobuf/text_format.py |
# Protocol Buffers - Google's data interchange format # Copyright 2008 Google Inc. All rights reserved. # https://developers.google.com/protocol-buffers/ # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above # copyright notice, this list of conditions and the following disclaimer # in the documentation and/or other materials provided with the # distribution. # * Neither the name of Google Inc. nor the names of its # contributors may be used to endorse or promote products derived from # this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Contains routines for printing protocol messages in text format. Simple usage example: # Create a proto object and serialize it to a text proto string. message = my_proto_pb2.MyMessage(foo='bar') text_proto = text_format.MessageToString(message) # Parse a text proto string. message = text_format.Parse(text_proto, my_proto_pb2.MyMessage()) """ __author__ = 'kenton@google.com (Kenton Varda)' import io import re import six if six.PY3: long = int # pylint: disable=redefined-builtin,invalid-name # pylint: disable=g-import-not-at-top from google.protobuf.internal import type_checkers from google.protobuf import descriptor from google.protobuf import text_encoding __all__ = ['MessageToString', 'PrintMessage', 'PrintField', 'PrintFieldValue', 'Merge'] _INTEGER_CHECKERS = (type_checkers.Uint32ValueChecker(), type_checkers.Int32ValueChecker(), type_checkers.Uint64ValueChecker(), type_checkers.Int64ValueChecker()) _FLOAT_INFINITY = re.compile('-?inf(?:inity)?f?', re.IGNORECASE) _FLOAT_NAN = re.compile('nanf?', re.IGNORECASE) _FLOAT_TYPES = frozenset([descriptor.FieldDescriptor.CPPTYPE_FLOAT, descriptor.FieldDescriptor.CPPTYPE_DOUBLE]) _QUOTES = frozenset(("'", '"')) _ANY_FULL_TYPE_NAME = 'google.protobuf.Any' class Error(Exception): """Top-level module error for text_format.""" class ParseError(Error): """Thrown in case of text parsing or tokenizing error.""" def __init__(self, message=None, line=None, column=None): if message is not None and line is not None: loc = str(line) if column is not None: loc += ':{0}'.format(column) message = '{0} : {1}'.format(loc, message) if message is not None: super(ParseError, self).__init__(message) else: super(ParseError, self).__init__() self._line = line self._column = column def GetLine(self): return self._line def GetColumn(self): return self._column class TextWriter(object): def __init__(self, as_utf8): if six.PY2: self._writer = io.BytesIO() else: self._writer = io.StringIO() def write(self, val): if six.PY2: if isinstance(val, six.text_type): val = val.encode('utf-8') return self._writer.write(val) def close(self): return self._writer.close() def getvalue(self): return self._writer.getvalue() def MessageToString(message, as_utf8=False, as_one_line=False, pointy_brackets=False, use_index_order=False, float_format=None, use_field_number=False, descriptor_pool=None, indent=0, message_formatter=None): """Convert protobuf message to text format. Floating point values can be formatted compactly with 15 digits of precision (which is the most that IEEE 754 "double" can guarantee) using float_format='.15g'. To ensure that converting to text and back to a proto will result in an identical value, float_format='.17g' should be used. Args: message: The protocol buffers message. as_utf8: Produce text output in UTF8 format. as_one_line: Don't introduce newlines between fields. pointy_brackets: If True, use angle brackets instead of curly braces for nesting. use_index_order: If True, print fields of a proto message using the order defined in source code instead of the field number. By default, use the field number order. float_format: If set, use this to specify floating point number formatting (per the "Format Specification Mini-Language"); otherwise, str() is used. use_field_number: If True, print field numbers instead of names. descriptor_pool: A DescriptorPool used to resolve Any types. indent: The indent level, in terms of spaces, for pretty print. message_formatter: A function(message, indent, as_one_line): unicode|None to custom format selected sub-messages (usually based on message type). Use to pretty print parts of the protobuf for easier diffing. Returns: A string of the text formatted protocol buffer message. """ out = TextWriter(as_utf8) printer = _Printer(out, indent, as_utf8, as_one_line, pointy_brackets, use_index_order, float_format, use_field_number, descriptor_pool, message_formatter) printer.PrintMessage(message) result = out.getvalue() out.close() if as_one_line: return result.rstrip() return result def _IsMapEntry(field): return (field.type == descriptor.FieldDescriptor.TYPE_MESSAGE and field.message_type.has_options and field.message_type.GetOptions().map_entry) def PrintMessage(message, out, indent=0, as_utf8=False, as_one_line=False, pointy_brackets=False, use_index_order=False, float_format=None, use_field_number=False, descriptor_pool=None, message_formatter=None): printer = _Printer(out, indent, as_utf8, as_one_line, pointy_brackets, use_index_order, float_format, use_field_number, descriptor_pool, message_formatter) printer.PrintMessage(message) def PrintField(field, value, out, indent=0, as_utf8=False, as_one_line=False, pointy_brackets=False, use_index_order=False, float_format=None, message_formatter=None): """Print a single field name/value pair.""" printer = _Printer(out, indent, as_utf8, as_one_line, pointy_brackets, use_index_order, float_format, message_formatter) printer.PrintField(field, value) def PrintFieldValue(field, value, out, indent=0, as_utf8=False, as_one_line=False, pointy_brackets=False, use_index_order=False, float_format=None, message_formatter=None): """Print a single field value (not including name).""" printer = _Printer(out, indent, as_utf8, as_one_line, pointy_brackets, use_index_order, float_format, message_formatter) printer.PrintFieldValue(field, value) def _BuildMessageFromTypeName(type_name, descriptor_pool): """Returns a protobuf message instance. Args: type_name: Fully-qualified protobuf message type name string. descriptor_pool: DescriptorPool instance. Returns: A Message instance of type matching type_name, or None if the a Descriptor wasn't found matching type_name. """ # pylint: disable=g-import-not-at-top if descriptor_pool is None: from google.protobuf import descriptor_pool as pool_mod descriptor_pool = pool_mod.Default() from google.protobuf import symbol_database database = symbol_database.Default() try: message_descriptor = descriptor_pool.FindMessageTypeByName(type_name) except KeyError: return None message_type = database.GetPrototype(message_descriptor) return message_type() class _Printer(object): """Text format printer for protocol message.""" def __init__(self, out, indent=0, as_utf8=False, as_one_line=False, pointy_brackets=False, use_index_order=False, float_format=None, use_field_number=False, descriptor_pool=None, message_formatter=None): """Initialize the Printer. Floating point values can be formatted compactly with 15 digits of precision (which is the most that IEEE 754 "double" can guarantee) using float_format='.15g'. To ensure that converting to text and back to a proto will result in an identical value, float_format='.17g' should be used. Args: out: To record the text format result. indent: The indent level for pretty print. as_utf8: Produce text output in UTF8 format. as_one_line: Don't introduce newlines between fields. pointy_brackets: If True, use angle brackets instead of curly braces for nesting. use_index_order: If True, print fields of a proto message using the order defined in source code instead of the field number. By default, use the field number order. float_format: If set, use this to specify floating point number formatting (per the "Format Specification Mini-Language"); otherwise, str() is used. use_field_number: If True, print field numbers instead of names. descriptor_pool: A DescriptorPool used to resolve Any types. message_formatter: A function(message, indent, as_one_line): unicode|None to custom format selected sub-messages (usually based on message type). Use to pretty print parts of the protobuf for easier diffing. """ self.out = out self.indent = indent self.as_utf8 = as_utf8 self.as_one_line = as_one_line self.pointy_brackets = pointy_brackets self.use_index_order = use_index_order self.float_format = float_format self.use_field_number = use_field_number self.descriptor_pool = descriptor_pool self.message_formatter = message_formatter def _TryPrintAsAnyMessage(self, message): """Serializes if message is a google.protobuf.Any field.""" packed_message = _BuildMessageFromTypeName(message.TypeName(), self.descriptor_pool) if packed_message: packed_message.MergeFromString(message.value) self.out.write('%s[%s]' % (self.indent * ' ', message.type_url)) self._PrintMessageFieldValue(packed_message) self.out.write(' ' if self.as_one_line else '\n') return True else: return False def _TryCustomFormatMessage(self, message): formatted = self.message_formatter(message, self.indent, self.as_one_line) if formatted is None: return False out = self.out out.write(' ' * self.indent) out.write(formatted) out.write(' ' if self.as_one_line else '\n') return True def PrintMessage(self, message): """Convert protobuf message to text format. Args: message: The protocol buffers message. """ if self.message_formatter and self._TryCustomFormatMessage(message): return if (message.DESCRIPTOR.full_name == _ANY_FULL_TYPE_NAME and self._TryPrintAsAnyMessage(message)): return fields = message.ListFields() if self.use_index_order: fields.sort(key=lambda x: x[0].index) for field, value in fields: if _IsMapEntry(field): for key in sorted(value): # This is slow for maps with submessage entires because it copies the # entire tree. Unfortunately this would take significant refactoring # of this file to work around. # # TODO(haberman): refactor and optimize if this becomes an issue. entry_submsg = value.GetEntryClass()(key=key, value=value[key]) self.PrintField(field, entry_submsg) elif field.label == descriptor.FieldDescriptor.LABEL_REPEATED: for element in value: self.PrintField(field, element) else: self.PrintField(field, value) def PrintField(self, field, value): """Print a single field name/value pair.""" out = self.out out.write(' ' * self.indent) if self.use_field_number: out.write(str(field.number)) else: if field.is_extension: out.write('[') if (field.containing_type.GetOptions().message_set_wire_format and field.type == descriptor.FieldDescriptor.TYPE_MESSAGE and field.label == descriptor.FieldDescriptor.LABEL_OPTIONAL): out.write(field.message_type.full_name) else: out.write(field.full_name) out.write(']') elif field.type == descriptor.FieldDescriptor.TYPE_GROUP: # For groups, use the capitalized name. out.write(field.message_type.name) else: out.write(field.name) if field.cpp_type != descriptor.FieldDescriptor.CPPTYPE_MESSAGE: # The colon is optional in this case, but our cross-language golden files # don't include it. out.write(': ') self.PrintFieldValue(field, value) if self.as_one_line: out.write(' ') else: out.write('\n') def _PrintMessageFieldValue(self, value): if self.pointy_brackets: openb = '<' closeb = '>' else: openb = '{' closeb = '}' if self.as_one_line: self.out.write(' %s ' % openb) self.PrintMessage(value) self.out.write(closeb) else: self.out.write(' %s\n' % openb) self.indent += 2 self.PrintMessage(value) self.indent -= 2 self.out.write(' ' * self.indent + closeb) def PrintFieldValue(self, field, value): """Print a single field value (not including name). For repeated fields, the value should be a single element. Args: field: The descriptor of the field to be printed. value: The value of the field. """ out = self.out if field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE: self._PrintMessageFieldValue(value) elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_ENUM: enum_value = field.enum_type.values_by_number.get(value, None) if enum_value is not None: out.write(enum_value.name) else: out.write(str(value)) elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_STRING: out.write('\"') if isinstance(value, six.text_type): out_value = value.encode('utf-8') else: out_value = value if field.type == descriptor.FieldDescriptor.TYPE_BYTES: # We need to escape non-UTF8 chars in TYPE_BYTES field. out_as_utf8 = False else: out_as_utf8 = self.as_utf8 out.write(text_encoding.CEscape(out_value, out_as_utf8)) out.write('\"') elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_BOOL: if value: out.write('true') else: out.write('false') elif field.cpp_type in _FLOAT_TYPES and self.float_format is not None: out.write('{1:{0}}'.format(self.float_format, value)) else: out.write(str(value)) def Parse(text, message, allow_unknown_extension=False, allow_field_number=False, descriptor_pool=None): """Parses a text representation of a protocol message into a message. NOTE: for historical reasons this function does not clear the input message. This is different from what the binary msg.ParseFrom(...) does. Example a = MyProto() a.repeated_field.append('test') b = MyProto() text_format.Parse(repr(a), b) text_format.Parse(repr(a), b) # repeated_field contains ["test", "test"] # Binary version: b.ParseFromString(a.SerializeToString()) # repeated_field is now "test" Caller is responsible for clearing the message as needed. Args: text: Message text representation. message: A protocol buffer message to merge into. allow_unknown_extension: if True, skip over missing extensions and keep parsing allow_field_number: if True, both field number and field name are allowed. descriptor_pool: A DescriptorPool used to resolve Any types. Returns: The same message passed as argument. Raises: ParseError: On text parsing problems. """ if not isinstance(text, str): if six.PY3: text = text.decode('utf-8') else: text = text.encode('utf-8') return ParseLines(text.split('\n'), message, allow_unknown_extension, allow_field_number, descriptor_pool=descriptor_pool) def Merge(text, message, allow_unknown_extension=False, allow_field_number=False, descriptor_pool=None): """Parses a text representation of a protocol message into a message. Like Parse(), but allows repeated values for a non-repeated field, and uses the last one. Args: text: Message text representation. message: A protocol buffer message to merge into. allow_unknown_extension: if True, skip over missing extensions and keep parsing allow_field_number: if True, both field number and field name are allowed. descriptor_pool: A DescriptorPool used to resolve Any types. Returns: The same message passed as argument. Raises: ParseError: On text parsing problems. """ if not isinstance(text, str): if six.PY3: text = text.decode('utf-8') else: text = text.encode('utf-8') return MergeLines( text.split('\n'), message, allow_unknown_extension, allow_field_number, descriptor_pool=descriptor_pool) def ParseLines(lines, message, allow_unknown_extension=False, allow_field_number=False, descriptor_pool=None): """Parses a text representation of a protocol message into a message. Args: lines: An iterable of lines of a message's text representation. message: A protocol buffer message to merge into. allow_unknown_extension: if True, skip over missing extensions and keep parsing allow_field_number: if True, both field number and field name are allowed. descriptor_pool: A DescriptorPool used to resolve Any types. Returns: The same message passed as argument. Raises: ParseError: On text parsing problems. """ parser = _Parser(allow_unknown_extension, allow_field_number, descriptor_pool=descriptor_pool) return parser.ParseLines(lines, message) def MergeLines(lines, message, allow_unknown_extension=False, allow_field_number=False, descriptor_pool=None): """Parses a text representation of a protocol message into a message. Args: lines: An iterable of lines of a message's text representation. message: A protocol buffer message to merge into. allow_unknown_extension: if True, skip over missing extensions and keep parsing allow_field_number: if True, both field number and field name are allowed. descriptor_pool: A DescriptorPool used to resolve Any types. Returns: The same message passed as argument. Raises: ParseError: On text parsing problems. """ parser = _Parser(allow_unknown_extension, allow_field_number, descriptor_pool=descriptor_pool) return parser.MergeLines(lines, message) class _Parser(object): """Text format parser for protocol message.""" def __init__(self, allow_unknown_extension=False, allow_field_number=False, descriptor_pool=None): self.allow_unknown_extension = allow_unknown_extension self.allow_field_number = allow_field_number self.descriptor_pool = descriptor_pool def ParseFromString(self, text, message): """Parses a text representation of a protocol message into a message.""" if not isinstance(text, str): text = text.decode('utf-8') return self.ParseLines(text.split('\n'), message) def ParseLines(self, lines, message): """Parses a text representation of a protocol message into a message.""" self._allow_multiple_scalars = False self._ParseOrMerge(lines, message) return message def MergeFromString(self, text, message): """Merges a text representation of a protocol message into a message.""" return self._MergeLines(text.split('\n'), message) def MergeLines(self, lines, message): """Merges a text representation of a protocol message into a message.""" self._allow_multiple_scalars = True self._ParseOrMerge(lines, message) return message def _ParseOrMerge(self, lines, message): """Converts a text representation of a protocol message into a message. Args: lines: Lines of a message's text representation. message: A protocol buffer message to merge into. Raises: ParseError: On text parsing problems. """ tokenizer = Tokenizer(lines) while not tokenizer.AtEnd(): self._MergeField(tokenizer, message) def _MergeField(self, tokenizer, message): """Merges a single protocol message field into a message. Args: tokenizer: A tokenizer to parse the field name and values. message: A protocol message to record the data. Raises: ParseError: In case of text parsing problems. """ message_descriptor = message.DESCRIPTOR if tokenizer.TryConsume('['): name = [tokenizer.ConsumeIdentifier()] while tokenizer.TryConsume('.'): name.append(tokenizer.ConsumeIdentifier()) name = '.'.join(name) if not message_descriptor.is_extendable: raise tokenizer.ParseErrorPreviousToken( 'Message type "%s" does not have extensions.' % message_descriptor.full_name) # pylint: disable=protected-access field = message.Extensions._FindExtensionByName(name) # pylint: enable=protected-access if not field: if self.allow_unknown_extension: field = None else: raise tokenizer.ParseErrorPreviousToken( 'Extension "%s" not registered. ' 'Did you import the _pb2 module which defines it? ' 'If you are trying to place the extension in the MessageSet ' 'field of another message that is in an Any or MessageSet field, ' 'that message\'s _pb2 module must be imported as well' % name) elif message_descriptor != field.containing_type: raise tokenizer.ParseErrorPreviousToken( 'Extension "%s" does not extend message type "%s".' % (name, message_descriptor.full_name)) tokenizer.Consume(']') else: name = tokenizer.ConsumeIdentifierOrNumber() if self.allow_field_number and name.isdigit(): number = ParseInteger(name, True, True) field = message_descriptor.fields_by_number.get(number, None) if not field and message_descriptor.is_extendable: field = message.Extensions._FindExtensionByNumber(number) else: field = message_descriptor.fields_by_name.get(name, None) # Group names are expected to be capitalized as they appear in the # .proto file, which actually matches their type names, not their field # names. if not field: field = message_descriptor.fields_by_name.get(name.lower(), None) if field and field.type != descriptor.FieldDescriptor.TYPE_GROUP: field = None if (field and field.type == descriptor.FieldDescriptor.TYPE_GROUP and field.message_type.name != name): field = None if not field: raise tokenizer.ParseErrorPreviousToken( 'Message type "%s" has no field named "%s".' % (message_descriptor.full_name, name)) if field: if not self._allow_multiple_scalars and field.containing_oneof: # Check if there's a different field set in this oneof. # Note that we ignore the case if the same field was set before, and we # apply _allow_multiple_scalars to non-scalar fields as well. which_oneof = message.WhichOneof(field.containing_oneof.name) if which_oneof is not None and which_oneof != field.name: raise tokenizer.ParseErrorPreviousToken( 'Field "%s" is specified along with field "%s", another member ' 'of oneof "%s" for message type "%s".' % (field.name, which_oneof, field.containing_oneof.name, message_descriptor.full_name)) if field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE: tokenizer.TryConsume(':') merger = self._MergeMessageField else: tokenizer.Consume(':') merger = self._MergeScalarField if (field.label == descriptor.FieldDescriptor.LABEL_REPEATED and tokenizer.TryConsume('[')): # Short repeated format, e.g. "foo: [1, 2, 3]" while True: merger(tokenizer, message, field) if tokenizer.TryConsume(']'): break tokenizer.Consume(',') else: merger(tokenizer, message, field) else: # Proto field is unknown. assert self.allow_unknown_extension _SkipFieldContents(tokenizer) # For historical reasons, fields may optionally be separated by commas or # semicolons. if not tokenizer.TryConsume(','): tokenizer.TryConsume(';') def _ConsumeAnyTypeUrl(self, tokenizer): """Consumes a google.protobuf.Any type URL and returns the type name.""" # Consume "type.googleapis.com/". prefix = [tokenizer.ConsumeIdentifier()] tokenizer.Consume('.') prefix.append(tokenizer.ConsumeIdentifier()) tokenizer.Consume('.') prefix.append(tokenizer.ConsumeIdentifier()) tokenizer.Consume('/') # Consume the fully-qualified type name. name = [tokenizer.ConsumeIdentifier()] while tokenizer.TryConsume('.'): name.append(tokenizer.ConsumeIdentifier()) return '.'.join(prefix), '.'.join(name) def _MergeMessageField(self, tokenizer, message, field): """Merges a single scalar field into a message. Args: tokenizer: A tokenizer to parse the field value. message: The message of which field is a member. field: The descriptor of the field to be merged. Raises: ParseError: In case of text parsing problems. """ is_map_entry = _IsMapEntry(field) if tokenizer.TryConsume('<'): end_token = '>' else: tokenizer.Consume('{') end_token = '}' if (field.message_type.full_name == _ANY_FULL_TYPE_NAME and tokenizer.TryConsume('[')): type_url_prefix, packed_type_name = self._ConsumeAnyTypeUrl(tokenizer) tokenizer.Consume(']') tokenizer.TryConsume(':') if tokenizer.TryConsume('<'): expanded_any_end_token = '>' else: tokenizer.Consume('{') expanded_any_end_token = '}' expanded_any_sub_message = _BuildMessageFromTypeName(packed_type_name, self.descriptor_pool) if not expanded_any_sub_message: raise ParseError('Type %s not found in descriptor pool' % packed_type_name) while not tokenizer.TryConsume(expanded_any_end_token): if tokenizer.AtEnd(): raise tokenizer.ParseErrorPreviousToken('Expected "%s".' % (expanded_any_end_token,)) self._MergeField(tokenizer, expanded_any_sub_message) if field.label == descriptor.FieldDescriptor.LABEL_REPEATED: any_message = getattr(message, field.name).add() else: any_message = getattr(message, field.name) any_message.Pack(expanded_any_sub_message, type_url_prefix=type_url_prefix) elif field.label == descriptor.FieldDescriptor.LABEL_REPEATED: if field.is_extension: sub_message = message.Extensions[field].add() elif is_map_entry: sub_message = getattr(message, field.name).GetEntryClass()() else: sub_message = getattr(message, field.name).add() else: if field.is_extension: sub_message = message.Extensions[field] else: sub_message = getattr(message, field.name) sub_message.SetInParent() while not tokenizer.TryConsume(end_token): if tokenizer.AtEnd(): raise tokenizer.ParseErrorPreviousToken('Expected "%s".' % (end_token,)) self._MergeField(tokenizer, sub_message) if is_map_entry: value_cpptype = field.message_type.fields_by_name['value'].cpp_type if value_cpptype == descriptor.FieldDescriptor.CPPTYPE_MESSAGE: value = getattr(message, field.name)[sub_message.key] value.MergeFrom(sub_message.value) else: getattr(message, field.name)[sub_message.key] = sub_message.value @staticmethod def _IsProto3Syntax(message): message_descriptor = message.DESCRIPTOR return (hasattr(message_descriptor, 'syntax') and message_descriptor.syntax == 'proto3') def _MergeScalarField(self, tokenizer, message, field): """Merges a single scalar field into a message. Args: tokenizer: A tokenizer to parse the field value. message: A protocol message to record the data. field: The descriptor of the field to be merged. Raises: ParseError: In case of text parsing problems. RuntimeError: On runtime errors. """ _ = self.allow_unknown_extension value = None if field.type in (descriptor.FieldDescriptor.TYPE_INT32, descriptor.FieldDescriptor.TYPE_SINT32, descriptor.FieldDescriptor.TYPE_SFIXED32): value = _ConsumeInt32(tokenizer) elif field.type in (descriptor.FieldDescriptor.TYPE_INT64, descriptor.FieldDescriptor.TYPE_SINT64, descriptor.FieldDescriptor.TYPE_SFIXED64): value = _ConsumeInt64(tokenizer) elif field.type in (descriptor.FieldDescriptor.TYPE_UINT32, descriptor.FieldDescriptor.TYPE_FIXED32): value = _ConsumeUint32(tokenizer) elif field.type in (descriptor.FieldDescriptor.TYPE_UINT64, descriptor.FieldDescriptor.TYPE_FIXED64): value = _ConsumeUint64(tokenizer) elif field.type in (descriptor.FieldDescriptor.TYPE_FLOAT, descriptor.FieldDescriptor.TYPE_DOUBLE): value = tokenizer.ConsumeFloat() elif field.type == descriptor.FieldDescriptor.TYPE_BOOL: value = tokenizer.ConsumeBool() elif field.type == descriptor.FieldDescriptor.TYPE_STRING: value = tokenizer.ConsumeString() elif field.type == descriptor.FieldDescriptor.TYPE_BYTES: value = tokenizer.ConsumeByteString() elif field.type == descriptor.FieldDescriptor.TYPE_ENUM: value = tokenizer.ConsumeEnum(field) else: raise RuntimeError('Unknown field type %d' % field.type) if field.label == descriptor.FieldDescriptor.LABEL_REPEATED: if field.is_extension: message.Extensions[field].append(value) else: getattr(message, field.name).append(value) else: # Proto3 doesn't represent presence so we can't test if multiple scalars # have occurred. We have to allow them. can_check_presence = not self._IsProto3Syntax(message) if field.is_extension: if (not self._allow_multiple_scalars and can_check_presence and message.HasExtension(field)): raise tokenizer.ParseErrorPreviousToken( 'Message type "%s" should not have multiple "%s" extensions.' % (message.DESCRIPTOR.full_name, field.full_name)) else: message.Extensions[field] = value else: if (not self._allow_multiple_scalars and can_check_presence and message.HasField(field.name)): raise tokenizer.ParseErrorPreviousToken( 'Message type "%s" should not have multiple "%s" fields.' % (message.DESCRIPTOR.full_name, field.name)) else: setattr(message, field.name, value) def _SkipFieldContents(tokenizer): """Skips over contents (value or message) of a field. Args: tokenizer: A tokenizer to parse the field name and values. """ # Try to guess the type of this field. # If this field is not a message, there should be a ":" between the # field name and the field value and also the field value should not # start with "{" or "<" which indicates the beginning of a message body. # If there is no ":" or there is a "{" or "<" after ":", this field has # to be a message or the input is ill-formed. if tokenizer.TryConsume(':') and not tokenizer.LookingAt( '{') and not tokenizer.LookingAt('<'): _SkipFieldValue(tokenizer) else: _SkipFieldMessage(tokenizer) def _SkipField(tokenizer): """Skips over a complete field (name and value/message). Args: tokenizer: A tokenizer to parse the field name and values. """ if tokenizer.TryConsume('['): # Consume extension name. tokenizer.ConsumeIdentifier() while tokenizer.TryConsume('.'): tokenizer.ConsumeIdentifier() tokenizer.Consume(']') else: tokenizer.ConsumeIdentifier() _SkipFieldContents(tokenizer) # For historical reasons, fields may optionally be separated by commas or # semicolons. if not tokenizer.TryConsume(','): tokenizer.TryConsume(';') def _SkipFieldMessage(tokenizer): """Skips over a field message. Args: tokenizer: A tokenizer to parse the field name and values. """ if tokenizer.TryConsume('<'): delimiter = '>' else: tokenizer.Consume('{') delimiter = '}' while not tokenizer.LookingAt('>') and not tokenizer.LookingAt('}'): _SkipField(tokenizer) tokenizer.Consume(delimiter) def _SkipFieldValue(tokenizer): """Skips over a field value. Args: tokenizer: A tokenizer to parse the field name and values. Raises: ParseError: In case an invalid field value is found. """ # String/bytes tokens can come in multiple adjacent string literals. # If we can consume one, consume as many as we can. if tokenizer.TryConsumeByteString(): while tokenizer.TryConsumeByteString(): pass return if (not tokenizer.TryConsumeIdentifier() and not _TryConsumeInt64(tokenizer) and not _TryConsumeUint64(tokenizer) and not tokenizer.TryConsumeFloat()): raise ParseError('Invalid field value: ' + tokenizer.token) class Tokenizer(object): """Protocol buffer text representation tokenizer. This class handles the lower level string parsing by splitting it into meaningful tokens. It was directly ported from the Java protocol buffer API. """ _WHITESPACE = re.compile(r'\s+') _COMMENT = re.compile(r'(\s*#.*$)', re.MULTILINE) _WHITESPACE_OR_COMMENT = re.compile(r'(\s|(#.*$))+', re.MULTILINE) _TOKEN = re.compile('|'.join([ r'[a-zA-Z_][0-9a-zA-Z_+-]*', # an identifier r'([0-9+-]|(\.[0-9]))[0-9a-zA-Z_.+-]*', # a number ] + [ # quoted str for each quote mark r'{qt}([^{qt}\n\\]|\\.)*({qt}|\\?$)'.format(qt=mark) for mark in _QUOTES ])) _IDENTIFIER = re.compile(r'[^\d\W]\w*') _IDENTIFIER_OR_NUMBER = re.compile(r'\w+') def __init__(self, lines, skip_comments=True): self._position = 0 self._line = -1 self._column = 0 self._token_start = None self.token = '' self._lines = iter(lines) self._current_line = '' self._previous_line = 0 self._previous_column = 0 self._more_lines = True self._skip_comments = skip_comments self._whitespace_pattern = (skip_comments and self._WHITESPACE_OR_COMMENT or self._WHITESPACE) self._SkipWhitespace() self.NextToken() def LookingAt(self, token): return self.token == token def AtEnd(self): """Checks the end of the text was reached. Returns: True iff the end was reached. """ return not self.token def _PopLine(self): while len(self._current_line) <= self._column: try: self._current_line = next(self._lines) except StopIteration: self._current_line = '' self._more_lines = False return else: self._line += 1 self._column = 0 def _SkipWhitespace(self): while True: self._PopLine() match = self._whitespace_pattern.match(self._current_line, self._column) if not match: break length = len(match.group(0)) self._column += length def TryConsume(self, token): """Tries to consume a given piece of text. Args: token: Text to consume. Returns: True iff the text was consumed. """ if self.token == token: self.NextToken() return True return False def Consume(self, token): """Consumes a piece of text. Args: token: Text to consume. Raises: ParseError: If the text couldn't be consumed. """ if not self.TryConsume(token): raise self.ParseError('Expected "%s".' % token) def ConsumeComment(self): result = self.token if not self._COMMENT.match(result): raise self.ParseError('Expected comment.') self.NextToken() return result def ConsumeCommentOrTrailingComment(self): """Consumes a comment, returns a 2-tuple (trailing bool, comment str).""" # Tokenizer initializes _previous_line and _previous_column to 0. As the # tokenizer starts, it looks like there is a previous token on the line. just_started = self._line == 0 and self._column == 0 before_parsing = self._previous_line comment = self.ConsumeComment() # A trailing comment is a comment on the same line than the previous token. trailing = (self._previous_line == before_parsing and not just_started) return trailing, comment def TryConsumeIdentifier(self): try: self.ConsumeIdentifier() return True except ParseError: return False def ConsumeIdentifier(self): """Consumes protocol message field identifier. Returns: Identifier string. Raises: ParseError: If an identifier couldn't be consumed. """ result = self.token if not self._IDENTIFIER.match(result): raise self.ParseError('Expected identifier.') self.NextToken() return result def TryConsumeIdentifierOrNumber(self): try: self.ConsumeIdentifierOrNumber() return True except ParseError: return False def ConsumeIdentifierOrNumber(self): """Consumes protocol message field identifier. Returns: Identifier string. Raises: ParseError: If an identifier couldn't be consumed. """ result = self.token if not self._IDENTIFIER_OR_NUMBER.match(result): raise self.ParseError('Expected identifier or number, got %s.' % result) self.NextToken() return result def TryConsumeInteger(self): try: # Note: is_long only affects value type, not whether an error is raised. self.ConsumeInteger() return True except ParseError: return False def ConsumeInteger(self, is_long=False): """Consumes an integer number. Args: is_long: True if the value should be returned as a long integer. Returns: The integer parsed. Raises: ParseError: If an integer couldn't be consumed. """ try: result = _ParseAbstractInteger(self.token, is_long=is_long) except ValueError as e: raise self.ParseError(str(e)) self.NextToken() return result def TryConsumeFloat(self): try: self.ConsumeFloat() return True except ParseError: return False def ConsumeFloat(self): """Consumes an floating point number. Returns: The number parsed. Raises: ParseError: If a floating point number couldn't be consumed. """ try: result = ParseFloat(self.token) except ValueError as e: raise self.ParseError(str(e)) self.NextToken() return result def ConsumeBool(self): """Consumes a boolean value. Returns: The bool parsed. Raises: ParseError: If a boolean value couldn't be consumed. """ try: result = ParseBool(self.token) except ValueError as e: raise self.ParseError(str(e)) self.NextToken() return result def TryConsumeByteString(self): try: self.ConsumeByteString() return True except ParseError: return False def ConsumeString(self): """Consumes a string value. Returns: The string parsed. Raises: ParseError: If a string value couldn't be consumed. """ the_bytes = self.ConsumeByteString() try: return six.text_type(the_bytes, 'utf-8') except UnicodeDecodeError as e: raise self._StringParseError(e) def ConsumeByteString(self): """Consumes a byte array value. Returns: The array parsed (as a string). Raises: ParseError: If a byte array value couldn't be consumed. """ the_list = [self._ConsumeSingleByteString()] while self.token and self.token[0] in _QUOTES: the_list.append(self._ConsumeSingleByteString()) return b''.join(the_list) def _ConsumeSingleByteString(self): """Consume one token of a string literal. String literals (whether bytes or text) can come in multiple adjacent tokens which are automatically concatenated, like in C or Python. This method only consumes one token. Returns: The token parsed. Raises: ParseError: When the wrong format data is found. """ text = self.token if len(text) < 1 or text[0] not in _QUOTES: raise self.ParseError('Expected string but found: %r' % (text,)) if len(text) < 2 or text[-1] != text[0]: raise self.ParseError('String missing ending quote: %r' % (text,)) try: result = text_encoding.CUnescape(text[1:-1]) except ValueError as e: raise self.ParseError(str(e)) self.NextToken() return result def ConsumeEnum(self, field): try: result = ParseEnum(field, self.token) except ValueError as e: raise self.ParseError(str(e)) self.NextToken() return result def ParseErrorPreviousToken(self, message): """Creates and *returns* a ParseError for the previously read token. Args: message: A message to set for the exception. Returns: A ParseError instance. """ return ParseError(message, self._previous_line + 1, self._previous_column + 1) def ParseError(self, message): """Creates and *returns* a ParseError for the current token.""" return ParseError(message, self._line + 1, self._column + 1) def _StringParseError(self, e): return self.ParseError('Couldn\'t parse string: ' + str(e)) def NextToken(self): """Reads the next meaningful token.""" self._previous_line = self._line self._previous_column = self._column self._column += len(self.token) self._SkipWhitespace() if not self._more_lines: self.token = '' return match = self._TOKEN.match(self._current_line, self._column) if not match and not self._skip_comments: match = self._COMMENT.match(self._current_line, self._column) if match: token = match.group(0) self.token = token else: self.token = self._current_line[self._column] # Aliased so it can still be accessed by current visibility violators. # TODO(dbarnett): Migrate violators to textformat_tokenizer. _Tokenizer = Tokenizer # pylint: disable=invalid-name def _ConsumeInt32(tokenizer): """Consumes a signed 32bit integer number from tokenizer. Args: tokenizer: A tokenizer used to parse the number. Returns: The integer parsed. Raises: ParseError: If a signed 32bit integer couldn't be consumed. """ return _ConsumeInteger(tokenizer, is_signed=True, is_long=False) def _ConsumeUint32(tokenizer): """Consumes an unsigned 32bit integer number from tokenizer. Args: tokenizer: A tokenizer used to parse the number. Returns: The integer parsed. Raises: ParseError: If an unsigned 32bit integer couldn't be consumed. """ return _ConsumeInteger(tokenizer, is_signed=False, is_long=False) def _TryConsumeInt64(tokenizer): try: _ConsumeInt64(tokenizer) return True except ParseError: return False def _ConsumeInt64(tokenizer): """Consumes a signed 32bit integer number from tokenizer. Args: tokenizer: A tokenizer used to parse the number. Returns: The integer parsed. Raises: ParseError: If a signed 32bit integer couldn't be consumed. """ return _ConsumeInteger(tokenizer, is_signed=True, is_long=True) def _TryConsumeUint64(tokenizer): try: _ConsumeUint64(tokenizer) return True except ParseError: return False def _ConsumeUint64(tokenizer): """Consumes an unsigned 64bit integer number from tokenizer. Args: tokenizer: A tokenizer used to parse the number. Returns: The integer parsed. Raises: ParseError: If an unsigned 64bit integer couldn't be consumed. """ return _ConsumeInteger(tokenizer, is_signed=False, is_long=True) def _TryConsumeInteger(tokenizer, is_signed=False, is_long=False): try: _ConsumeInteger(tokenizer, is_signed=is_signed, is_long=is_long) return True except ParseError: return False def _ConsumeInteger(tokenizer, is_signed=False, is_long=False): """Consumes an integer number from tokenizer. Args: tokenizer: A tokenizer used to parse the number. is_signed: True if a signed integer must be parsed. is_long: True if a long integer must be parsed. Returns: The integer parsed. Raises: ParseError: If an integer with given characteristics couldn't be consumed. """ try: result = ParseInteger(tokenizer.token, is_signed=is_signed, is_long=is_long) except ValueError as e: raise tokenizer.ParseError(str(e)) tokenizer.NextToken() return result def ParseInteger(text, is_signed=False, is_long=False): """Parses an integer. Args: text: The text to parse. is_signed: True if a signed integer must be parsed. is_long: True if a long integer must be parsed. Returns: The integer value. Raises: ValueError: Thrown Iff the text is not a valid integer. """ # Do the actual parsing. Exception handling is propagated to caller. result = _ParseAbstractInteger(text, is_long=is_long) # Check if the integer is sane. Exceptions handled by callers. checker = _INTEGER_CHECKERS[2 * int(is_long) + int(is_signed)] checker.CheckValue(result) return result def _ParseAbstractInteger(text, is_long=False): """Parses an integer without checking size/signedness. Args: text: The text to parse. is_long: True if the value should be returned as a long integer. Returns: The integer value. Raises: ValueError: Thrown Iff the text is not a valid integer. """ # Do the actual parsing. Exception handling is propagated to caller. try: # We force 32-bit values to int and 64-bit values to long to make # alternate implementations where the distinction is more significant # (e.g. the C++ implementation) simpler. if is_long: return long(text, 0) else: return int(text, 0) except ValueError: raise ValueError('Couldn\'t parse integer: %s' % text) def ParseFloat(text): """Parse a floating point number. Args: text: Text to parse. Returns: The number parsed. Raises: ValueError: If a floating point number couldn't be parsed. """ try: # Assume Python compatible syntax. return float(text) except ValueError: # Check alternative spellings. if _FLOAT_INFINITY.match(text): if text[0] == '-': return float('-inf') else: return float('inf') elif _FLOAT_NAN.match(text): return float('nan') else: # assume '1.0f' format try: return float(text.rstrip('f')) except ValueError: raise ValueError('Couldn\'t parse float: %s' % text) def ParseBool(text): """Parse a boolean value. Args: text: Text to parse. Returns: Boolean values parsed Raises: ValueError: If text is not a valid boolean. """ if text in ('true', 't', '1', 'True'): return True elif text in ('false', 'f', '0', 'False'): return False else: raise ValueError('Expected "true" or "false".') def ParseEnum(field, value): """Parse an enum value. The value can be specified by a number (the enum value), or by a string literal (the enum name). Args: field: Enum field descriptor. value: String value. Returns: Enum value number. Raises: ValueError: If the enum value could not be parsed. """ enum_descriptor = field.enum_type try: number = int(value, 0) except ValueError: # Identifier. enum_value = enum_descriptor.values_by_name.get(value, None) if enum_value is None: raise ValueError('Enum type "%s" has no value named %s.' % (enum_descriptor.full_name, value)) else: # Numeric value. if hasattr(field.file, 'syntax'): # Attribute is checked for compatibility. if field.file.syntax == 'proto3': # Proto3 accept numeric unknown enums. return number enum_value = enum_descriptor.values_by_number.get(number, None) if enum_value is None: raise ValueError('Enum type "%s" has no value with number %d.' % (enum_descriptor.full_name, number)) return enum_value.number