语法分析

和通常的有语法的结构一样，proto的编译也经过词法(tokenize)和语法(parse)两个阶段，相关代码分别在tokenizer.cc和parser.cc两个文件中。

/// @file: protobuf\src\google\protobuf\compiler\parser.cc

bool Parser::ParseTopLevelStatement(FileDescriptorProto* file,

                                    const LocationRecorder& root_location) {

  if (TryConsumeEndOfDeclaration(";", NULL)) {

    // empty statement; ignore

    return true;

  } else if (LookingAt("message")) {

    LocationRecorder location(root_location,

                              FileDescriptorProto::kMessageTypeFieldNumber,

                              file->message_type_size());

    return ParseMessageDefinition(file->add_message_type(), location, file);

  } else if (LookingAt("enum")) {

    LocationRecorder location(root_location,

                              FileDescriptorProto::kEnumTypeFieldNumber,

                              file->enum_type_size());

    return ParseEnumDefinition(file->add_enum_type(), location, file);

  } else if (LookingAt("service")) {

    LocationRecorder location(root_location,

                              FileDescriptorProto::kServiceFieldNumber,

                              file->service_size());

    return ParseServiceDefinition(file->add_service(), location, file);

  } else if (LookingAt("extend")) {

    LocationRecorder location(root_location,

                              FileDescriptorProto::kExtensionFieldNumber);

    return ParseExtend(

        file->mutable_extension(), file->mutable_message_type(), root_location,

        FileDescriptorProto::kMessageTypeFieldNumber, location, file);

  } else if (LookingAt("import")) {

    return ParseImport(file->mutable_dependency(),

                       file->mutable_public_dependency(),

                       file->mutable_weak_dependency(), root_location, file);

  } else if (LookingAt("package")) {

    return ParsePackage(file, root_location, file);

  } else if (LookingAt("option")) {

    LocationRecorder location(root_location,

                              FileDescriptorProto::kOptionsFieldNumber);

    return ParseOption(file->mutable_options(), location, file,

                       OPTION_STATEMENT);

  } else {

    AddError("Expected top-level statement (e.g. \"message\").");

    return false;

  }

}

DescriptorTable

每个proto文件生成的cpp文件中都有一串特别醒目的字符串，该字符串位于一个DescriptorTable结构的const char* descriptor字段，它也是整个proto文件生成FileDescriptorProto结构序列化之后的内容。

/// @file: protobuf\src\google\protobuf\generated_message_reflection.h

// This struct tries to reduce unnecessary padding.

// The num_xxx might not be close to their respective pointer, but this saves

// padding.

struct PROTOBUF_EXPORT DescriptorTable {

  mutable bool is_initialized;

  bool is_eager;

  int size;  // of serialized descriptor

  const char* descriptor;

  const char* filename;

  once_flag* once;

  const DescriptorTable* const* deps;

  int num_deps;

  int num_messages;

  const MigrationSchema* schemas;

  const Message* const* default_instances;

  const uint32* offsets;

  // update the following descriptor arrays.

  Metadata* file_level_metadata;

  const EnumDescriptor** file_level_enum_descriptors;

  const ServiceDescriptor** file_level_service_descriptors;

};

生成的cpp代码

在生成的cpp文件中类似于2e这种莫名其妙的字符串是从哪里来的呢？例如这个变量名assign_descriptors_table_tsecer_2eproto中的2e，从代码实现看也很简单，就是把不属于数字字母(ascii_isalnum)的字符转换为""+16进制字符串。所以assign_descriptors_table_tsecer_2eproto中的"2e"就是由于"tsecer.proto"中的"."字符内码"2e"加上""组成；同样，如果是文件名中有""，那么会转换为"_5f"。

/// @file: protobuf\src\google\protobuf\compiler\cpp\cpp_helpers.cc

// Convert a file name into a valid identifier.

std::string FilenameIdentifier(const std::string& filename) {

  std::string result;

  for (int i = 0; i < filename.size(); i++) {

    if (ascii_isalnum(filename[i])) {

      result.push_back(filename[i]);

    } else {

      // Not alphanumeric.  To avoid any possibility of name conflicts we

      // use the hex code for the character.

      StrAppend(&result, "_",

                      strings::Hex(static_cast<uint8_t>(filename[i])));

    }

  }

  return result;

}

std::string UniqueName(const std::string& name, const std::string& filename,

                       const Options& options) {

  return name + "_" + FilenameIdentifier(filename);

}

///@file: protobuf\src\google\protobuf\compiler\cpp\cpp_file.cc

void FileGenerator::GenerateReflectionInitializationCode(io::Printer* printer) {

  Formatter format(printer, variables_);

///....

  // Embed the descriptor.  We simply serialize the entire

  // FileDescriptorProto/ and embed it as a string literal, which is parsed and

  // built into real descriptors at initialization time.

  const std::string protodef_name =

      UniqueName("descriptor_table_protodef", file_, options_);

///...

  // The DescriptorTable itself.

  // Should be "bool eager = NeedsEagerDescriptorAssignment(file_, options_);"

  // however this might cause a tsan failure in superroot b/148382879,

  // so disable for now.

  bool eager = false;

  format(

      "static ::$proto_ns$::internal::once_flag $desc_table$_once;\n"

      "const ::$proto_ns$::internal::DescriptorTable $desc_table$ = {\n"

      "  false, $1$, $2$, $3$, \"$filename$\", \n"

      "  &$desc_table$_once, $4$, $5$, $6$,\n"

      "  schemas, file_default_instances, $tablename$::offsets,\n"

      "  $7$, $file_level_enum_descriptors$, "

      "$file_level_service_descriptors$,\n"

      "};\n"

      // This function exists to be marked as weak.

      // It can significantly speed up compilation by breaking up LLVM's SCC in

      // the .pb.cc translation units. Large translation units see a reduction

      // of more than 35% of walltime for optimized builds.

      // Without the weak attribute all the messages in the file, including all

      // the vtables and everything they use become part of the same SCC through

      // a cycle like:

      // GetMetadata -> descriptor table -> default instances ->

      //   vtables -> GetMetadata

      // By adding a weak function here we break the connection from the

      // individual vtables back into the descriptor table.

      "PROTOBUF_ATTRIBUTE_WEAK const ::$proto_ns$::internal::DescriptorTable* "

      "$desc_table$_getter() {\n"

      "  return &$desc_table$;\n"

      "}\n"

      "\n",

      eager ? "true" : "false", file_data.size(), protodef_name,

      num_deps == 0 ? "nullptr" : variables_["desc_table"] + "_deps", num_deps,

      message_generators_.size(),

      message_generators_.empty() ? "nullptr"

                                  : variables_["file_level_metadata"]);

}

table的注册

既然源文件中有序列化之后的字符串，就可以在运行时把这个字符串反序列化成一个描述实例。

(gdb) bt

#0  google::protobuf::EncodedDescriptorDatabase::Add (this=0x61dc90, encoded_file_descriptor=0x7ffff7ab6e20 <descriptor_table_protodef_google_2fprotobuf_2fany_2eproto>, size=212)

    at google/protobuf/descriptor_database.cc:556

#1  0x00007ffff78f3486 in google::protobuf::DescriptorPool::InternalAddGeneratedFile (encoded_file_descriptor=0x7ffff7ab6e20 <descriptor_table_protodef_google_2fprotobuf_2fany_2eproto>,

    size=212) at google/protobuf/descriptor.cc:1357

#2  0x00007ffff79adc27 in google::protobuf::(anonymous namespace)::AddDescriptorsImpl (table=0x7ffff7dd8b80 <descriptor_table_google_2fprotobuf_2fany_2eproto>)

    at google/protobuf/generated_message_reflection.cc:2767

#3  0x00007ffff79adc60 in google::protobuf::(anonymous namespace)::AddDescriptors (table=0x7ffff7dd8b80 <descriptor_table_google_2fprotobuf_2fany_2eproto>)

    at google/protobuf/generated_message_reflection.cc:2778

#4  0x00007ffff79add3c in google::protobuf::internal::AddDescriptorsRunner::AddDescriptorsRunner (this=0x7ffff7dda284 <dynamic_init_dummy_google_2fprotobuf_2fany_2eproto>,

    table=0x7ffff7dd8b80 <descriptor_table_google_2fprotobuf_2fany_2eproto>) at google/protobuf/generated_message_reflection.cc:2813

#5  0x00007ffff78e885e in __static_initialization_and_destruction_0 (__initialize_p=1, __priority=102) at google/protobuf/any.pb.cc:76

#6  0x00007ffff78e8874 in _GLOBAL__sub_I.00102_any.pb.cc(void) () at ./google/protobuf/port_undef.inc:128

#7  0x00007ffff7dea4c3 in _dl_init_internal () from /lib64/ld-linux-x86-64.so.2

#8  0x00007ffff7ddc1aa in _dl_start_user () from /lib64/ld-linux-x86-64.so.2

#9  0x0000000000000001 in ?? ()

#10 0x00007fffffffe5c5 in ?? ()

#11 0x0000000000000000 in ?? ()

(gdb)

在生成的源代码中有一个全局变量

  // Force running AddDescriptors() at dynamic initialization time.

  PROTOBUF_ATTRIBUTE_INIT_PRIORITY static ::PROTOBUF_NAMESPACE_ID::internal::AddDescriptorsRunner dynamic_init_dummy_tsecer_2eproto(&descriptor_table_tsecer_2eproto);

会触发internal::AddDescriptorsRunner构造函数，进而触发下面的流程，并调用file.ParseFromArray来解析这个字符串来还原原始的proto文件内容。

/// @file: protobuf\src\google\protobuf\generated_message_reflection.cc

void AssignDescriptorsImpl(const DescriptorTable* table, bool eager) {

///...

  // Register the descriptor of this file.

  DescriptorPool::InternalAddGeneratedFile(table->descriptor, table->size);

  MessageFactory::InternalRegisterGeneratedFile(table);

}

/// @file: google/protobuf/descriptor_database.cc:556

bool EncodedDescriptorDatabase::Add(const void* encoded_file_descriptor,

                                    int size) {

  FileDescriptorProto file;

  if (file.ParseFromArray(encoded_file_descriptor, size)) {

    return index_->AddFile(file, std::make_pair(encoded_file_descriptor, size));

  } else {

    GOOGLE_LOG(ERROR) << "Invalid file descriptor data passed to "

                  "EncodedDescriptorDatabase::Add().";

    return false;

  }

}

pb内部Descriptor的proto

在pb内部，把proto文件本身的message也定义成了proto文件。要注意的是proto文件不是通过这个描述自动解析的，这里只是通过定义生成了指定格式的容器和对应的操作接口，另外，由于使用proto描述，所以也自动拥有了proto的序列化、反序列化两个重要的基础功能。

// @file: protobuf\src\google\protobufdescriptor.proto

// Describes a field within a message.

// Describes a field within a message.

message FieldDescriptorProto {

  enum Type {

    // 0 is reserved for errors.

    // Order is weird for historical reasons.

    TYPE_DOUBLE = 1;

    TYPE_FLOAT = 2;

    // Not ZigZag encoded.  Negative numbers take 10 bytes.  Use TYPE_SINT64 if

    // negative values are likely.

    TYPE_INT64 = 3;

    TYPE_UINT64 = 4;

    // Not ZigZag encoded.  Negative numbers take 10 bytes.  Use TYPE_SINT32 if

    // negative values are likely.

    TYPE_INT32 = 5;

    TYPE_FIXED64 = 6;

    TYPE_FIXED32 = 7;

    TYPE_BOOL = 8;

    TYPE_STRING = 9;

    // Tag-delimited aggregate.

    // Group type is deprecated and not supported in proto3. However, Proto3

    // implementations should still be able to parse the group wire format and

    // treat group fields as unknown fields.

    TYPE_GROUP = 10;

    TYPE_MESSAGE = 11;  // Length-delimited aggregate.

    // New in version 2.

    TYPE_BYTES = 12;

    TYPE_UINT32 = 13;

    TYPE_ENUM = 14;

    TYPE_SFIXED32 = 15;

    TYPE_SFIXED64 = 16;

    TYPE_SINT32 = 17;  // Uses ZigZag encoding.

    TYPE_SINT64 = 18;  // Uses ZigZag encoding.

  }

  enum Label {

    // 0 is reserved for errors

    LABEL_OPTIONAL = 1;

    LABEL_REQUIRED = 2;

    LABEL_REPEATED = 3;

  }

  optional string name = 1;

  optional int32 number = 3;

  optional Label label = 4;

  // If type_name is set, this need not be set.  If both this and type_name

  // are set, this must be one of TYPE_ENUM, TYPE_MESSAGE or TYPE_GROUP.

  optional Type type = 5;

字符串可视化

这里为什么使用8进制而没有使用更容易阅读的16进制呢？从代码上猜测：如果是6进制就可以直接把数值操作(除法或者取模)的结果加上'0'来获得一个；如果使用16进制，这个结果可能要经过一个表格转换之类的(因为 '0' + 10 != 'A')。

/// @file: protobuf\src\google\protobuf\stubs\strutil.cc

// ----------------------------------------------------------------------

// Escapes 'src' using C-style escape sequences, and appends the escaped string

// to 'dest'. This version is faster than calling CEscapeInternal as it computes

// the required space using a lookup table, and also does not do any special

// handling for Hex or UTF-8 characters.

// ----------------------------------------------------------------------

void CEscapeAndAppend(StringPiece src, std::string *dest) {

  size_t escaped_len = CEscapedLength(src);

  if (escaped_len == src.size()) {

    dest->append(src.data(), src.size());

    return;

  }

  size_t cur_dest_len = dest->size();

  dest->resize(cur_dest_len + escaped_len);

  char* append_ptr = &(*dest)[cur_dest_len];

  for (StringPiece::size_type i = 0; i < src.size(); ++i) {

    unsigned char c = static_cast<unsigned char>(src[i]);

    switch (c) {

      case '\n': *append_ptr++ = '\\'; *append_ptr++ = 'n'; break;

      case '\r': *append_ptr++ = '\\'; *append_ptr++ = 'r'; break;

      case '\t': *append_ptr++ = '\\'; *append_ptr++ = 't'; break;

      case '\"': *append_ptr++ = '\\'; *append_ptr++ = '\"'; break;

      case '\'': *append_ptr++ = '\\'; *append_ptr++ = '\''; break;

      case '\\': *append_ptr++ = '\\'; *append_ptr++ = '\\'; break;

      default:

        if (!isprint(c)) {

          *append_ptr++ = '\\';

          *append_ptr++ = '0' + c / 64;

          *append_ptr++ = '0' + (c % 64) / 8;

          *append_ptr++ = '0' + c % 8;

        } else {

          *append_ptr++ = c;

        }

        break;

    }

  }

}

栗子

描述文件

tsecer@harry: cat  tsecer.proto

syntax = "proto3";

message tsecer {

bytes harry = 1;

int32   fry   = 3;

};

tsecer@harry:

左括弧

在字符串描述中，经常会遇到醒目的左括弧字符，它是从哪里来？

其实这个是有前面

optional Type type = 5;

这个字段生成的：

由于'('字符的十六进制内码是0x28，对应的二进制是0b00101000，结合protobuf的编码格式。最低3个bit值为0，表示类型是一个VARINT；剩余内容为5，对应这里type字段的tag值为5。这个结果刚好是左括号而已。在左括弧之后，就是type的具体值了。在开始的例子中，输出中有两个左括弧，分别是"(\014"和"(\005"，括弧之后的八进制转换为10进制之后分别为15和5。

const char descriptor_table_protodef_tsecer_2eproto[] =

  "\n\014tsecer.proto\"$\n\006tsecer\022\r\n\005harry\030\001 \001(\014\022"

  "\013\n\003fry\030\003 \001(\005b\006proto3"

  ;

对应前面enum Type枚举类型，分别对应

    TYPE_INT32 = 5;

    TYPE_BYTES = 12;

对应，也和proto文件中的声明一致。

descriptor.cc和descriptor.pb.cc

descriptor.pb.cc是protoc根据proto文件自动生成的，而descriptor.cc则是人工编写的代码。

///@file: protobuf\src\google\protobuf\descriptor.cc

// Protocol Buffers - Google's data interchange format

// Copyright 2008 Google Inc.  All rights reserved.

// https://developers.google.com/protocol-buffers/

//

// Redistribution and use in source and binary forms, with or without

// modification, are permitted provided that the following conditions are

// met:

descriptor.pb.cc只是作为一个解析之后的容器，它的数据也是有编译器在完成proto语法分析之后添加的。

在descriptor.cc文件中(人工)实现了DebugString接口，所以descriptor经过DebugString输出的格式跟常规Message输出的格式并不一样。

tsecer@harry: cat main.cpp

#include <iostream>

#include <fstream>

#include <string>

#include "tsecer.pb.h"

#include <google/protobuf/util/json_util.h>

using namespace std;

int main(int argc, char* argv[]) {

    char str[] = "\00\01\02\03HelloWorld";

tsecer msg;

auto desc = msg.GetDescriptor();

printf("%s\n", desc->DebugString ().c_str());

msg.set_harry(std::string(str, sizeof(str)));

printf("%s\n", msg.DebugString().c_str());

return 0;

}

tsecer@harry: ./a.out

message tsecer {

  bytes harry = 1;

  int32 fry = 3;

}

harry: "\000\001\002\003HelloWorld\000"

tsecer@harry:

巴特西

protobuf如何还原proto源文件及描述字符串中左括弧的意义