• 前向传播的特点
• 前向传播的实现
• DAG(有向无环图)构造过程

使用传统的BP算法进行CNN训练时包括两个阶段：前向传播计算（Forward)和反向传播计算（Backward)。今天我们将注意力放在前向传播阶段。

前向传播阶段在实际应用中最常见，比如大量的在线系统（语音识别、文字识别、图像分类和检索等)都是仅前向传播阶段的应用;一些嵌入式系统（视觉机器人、无人机、智能语音 机器人）受限于计算资源，仅实现前向传播阶段，而反向传播计算则由计算性能更强大的服务器完成.

前向传播的特点

在前向传播阶段，数据源起于数据读取层，经过若干处理层，到达最后一层(可能是损失 层或特征层）。

网络中的权值在前向传播阶段不发生变化，可以看作常量。

网络路径是一个有向无环图（DirectedAcyclineGraph，DAG)。从最初的节点出发，经历若干处理层，不存在循环结构，因此数据流会直向前推进到达终点。

我们可以使用数据流分析方法对前向传播过程进行研究：

从输入数据集中取一个样本\((X,Y)\),其中X为数据，Y为标签。将X送入网络,逐层计算,得到相应的网络处理输出\(O\)。网络执行的计算可以用公式表达为：
\[
O = F_n(...(F_2(F_1(XW_1)W_2)...)W_n)
\]

其中,\(F_i ,i=1,2,...n\)表示非线性变换，而\(W_i=1,2,…n\),表示各个权值层权值。

得到网络输出\(O\)后，可以用\((Y,O)\)评估网络质量。理想的网络满足\(Y==O\)。

前向传播的实现

在Caffe中CNN前向传播过程由Net + Layer组合完成，中间结果和最终结果则使用Blob承载。下面我们深入代码来观察这一过程。

DAG(有向无环图)构造过程

首先我们从Net构造函数开始.

//从NetParameter对象构造
template <typename Dtype>
Net<Dtype>::Net(const NetParameter& param) {
  Init(param);
}

//从net.prototxt文件构造
template <typename Dtype>
Net<Dtype>::Net(const string& param_file, Phase phase,
    const int level, const vector<string>* stages) {
  NetParameter param;
  ReadNetParamsFromTextFileOrDie(param_file, &param);
  // Set phase, stages and level
  param.mutable_state()->set_phase(phase);
  if (stages != NULL) {
    for (int i = 0; i < stages->size(); i++) {
      param.mutable_state()->add_stage((*stages)[i]);
    }
  }
  param.mutable_state()->set_level(level);
  Init(param);
}

从上面的构造函数看到，二者都调用了Init()函数。传递给该函数的参数param是 NetParameter对象，我们已经之前的例程中使用过，了解过其数据结构描述(caffe.proto)。 我们可以从net.prototxt文件读取到内存中，初始化一个NetParameter对象，然后传递给Init()函数.

接着追踪Init()函数:

//这个函数很长
template <typename Dtype>
void Net<Dtype>::Init(const NetParameter& in_param) {
  // Set phase from the state.
  phase_ = in_param.state().phase();
  // Filter layers based on their include/exclude rules and
  // the current NetState.
  NetParameter filtered_param;
  //过滤一些参数,仅仅保留当前阶段参数.
  FilterNet(in_param, &filtered_param);
  LOG_IF(INFO, Caffe::root_solver())
      << "Initializing net from parameters: " << std::endl
      << filtered_param.DebugString();
  // Create a copy of filtered_param with splits added where necessary.(创建一个拷贝,之后就用这个拷贝)
  NetParameter param;
  InsertSplits(filtered_param, &param);
  // Basically, build all the layers and set up their connections.(构建所有Layer并将它们连接)
  name_ = param.name(); //网络名
  map<string, int> blob_name_to_idx;    //Blob名与索引的映射
  set<string> available_blobs;  //已有Blob名集合
  memory_used_ = 0;     //统计内存占用
  // For each layer, set up its input and output
  //对每个 Layer 设置输入 Blob (BottomBlob)和输出 Blob (TopBlob)
  bottom_vecs_.resize(param.layer_size()); //有多少层，就有多少个输入 Blob 
  top_vecs_.resize(param.layer_size()); //有多少层，就有多少个输出Blob 
  bottom_id_vecs_.resize(param.layer_size()); //记录每个层的输入Blob索引
  param_id_vecs_.resize(param.layer_size());    // 记录每个层的权值Blob索引
  top_id_vecs_.resize(param.layer_size());  // 记录每个层的输出Blob索引
  bottom_need_backward_.resize(param.layer_size()); //记录每个Blob是否需要反向传播过程
  
  //遍历每个层
  for (int layer_id = 0; layer_id < param.layer_size(); ++layer_id) {
    // Inherit phase from net if unset.(每个层的阶段标记.如果在层描述中未指定阶段，就使用Net的阶段)
    if (!param.layer(layer_id).has_phase()) {
      param.mutable_layer(layer_id)->set_phase(phase_);
    }
    // Setup layer.
    //获取层参数
    const LayerParameter& layer_param = param.layer(layer_id);
    if (layer_param.propagate_down_size() > 0) {
      CHECK_EQ(layer_param.propagate_down_size(),
          layer_param.bottom_size())
          << "propagate_down param must be specified "
          << "either 0 or bottom_size times ";
    }
    // Layer工厂，专业制造各种Layer，然后添加到Net类的layers_对象中 
    // 注意到这Layer的LayerParameter都继承自NetParameter
NetParameterlayers_.push_back(LayerRegistry<Dtype>::CreateLayer(layer_param));
    layer_names_.push_back(layer_param.name());
    LOG_IF(INFO, Caffe::root_solver())
        << "Creating Layer " << layer_param.name();
    bool need_backward = false;     //判断该层是否需要反向传播

    // Figure out this layer's input and output(确定该Layer的输入Blob和输出Blob)
    for (int bottom_id = 0; bottom_id < layer_param.bottom_size();
         ++bottom_id) {
         //遍历所有输入Blob,记录到Blob名集合、Blob名到索引映射中
      const int blob_id = AppendBottom(param, layer_id, bottom_id, &available_blobs, &blob_name_to_idx);
      // If a blob needs backward, this layer should provide it.
      need_backward |= blob_need_backward_[blob_id];
    }
    //输出Blob做同样的事
    int num_top = layer_param.top_size();
    for (int top_id = 0; top_id < num_top; ++top_id) {
      AppendTop(param, layer_id, top_id, &available_blobs, &blob_name_to_idx);
      // Collect Input layer tops as Net inputs.(收集输入层(InputLayer)信息，如果有，其输出blob将作为整个Net的输入)
      if (layer_param.type() == "Input") {
        const int blob_id = blobs_.size() - 1;
        net_input_blob_indices_.push_back(blob_id);
        net_input_blobs_.push_back(blobs_[blob_id].get());
      }
    }
    // If the layer specifies that AutoTopBlobs() -> true and the LayerParameter
    // specified fewer than the required number (as specified by
    // ExactNumTopBlobs() or MinTopBlobs()), allocate them here.
    Layer<Dtype>* layer = layers_[layer_id].get();
    if (layer->AutoTopBlobs()) {
      const int needed_num_top =
          std::max(layer->MinTopBlobs(), layer->ExactNumTopBlobs());
      for (; num_top < needed_num_top; ++num_top) {
        // Add "anonymous" top blobs -- do not modify available_blobs or
        // blob_name_to_idx as we don't want these blobs to be usable as input
        // to other layers.
        AppendTop(param, layer_id, num_top, NULL, NULL);
      }
    }
    
    
    // After this layer is connected, set it up.(Layer连接设置完毕，调用各个Layer的SetUp()函数)
    layers_[layer_id]->SetUp(bottom_vecs_[layer_id], top_vecs_[layer_id]);
    LOG_IF(INFO, Caffe::root_solver())
        << "Setting up " << layer_names_[layer_id];
        //设置输出Blob对损失函数的投票因子
    for (int top_id = 0; top_id < top_vecs_[layer_id].size(); ++top_id) {
      if (blob_loss_weights_.size() <= top_id_vecs_[layer_id][top_id]) {
        blob_loss_weights_.resize(top_id_vecs_[layer_id][top_id] + 1, Dtype(0));
      }
      blob_loss_weights_[top_id_vecs_[layer_id][top_id]] = layer->loss(top_id);
      //打印每层输出Blob尺寸信息
      LOG_IF(INFO, Caffe::root_solver())
          << "Top shape: " << top_vecs_[layer_id][top_id]->shape_string();
      if (layer->loss(top_id)) {
        LOG_IF(INFO, Caffe::root_solver())
            << "    with loss weight " << layer->loss(top_id);      //除了损失层的loss_weight为1,其它层都是0
      }
      //统计每个输出Blob内存占用量
      memory_used_ += top_vecs_[layer_id][top_id]->count();
    }
    //打印所有输出Blob内存占用量
    LOG_IF(INFO, Caffe::root_solver())
        << "Memory required for data: " << memory_used_ * sizeof(Dtype);
        
    //下面开始初始化各层权值Blob
    const int param_size = layer_param.param_size();
    const int num_param_blobs = layers_[layer_id]->blobs().size();
    //保证参数配置需要的权值Blob数目不大于实际对象的权值Blob数
    CHECK_LE(param_size, num_param_blobs)
        << "Too many params specified for layer " << layer_param.name();
    ParamSpec default_param_spec;
    //每个权值层(卷基层,全连接层)都要经历下面的过程
    for (int param_id = 0; param_id < num_param_blobs; ++param_id) {
      const ParamSpec* param_spec = (param_id < param_size) ?
          &layer_param.param(param_id) : &default_param_spec;
      const bool param_need_backward = param_spec->lr_mult() != 0;
      //设置权值层param(lr_mult:0)可以禁止其反向传播过程，即冻结权值
      need_backward |= param_need_backward;
      layers_[layer_id]->set_param_propagate_down(param_id,
                                                  param_need_backward);
    }
    for (int param_id = 0; param_id < num_param_blobs; ++param_id) {
    //记录权值Blob到Net后台数据库
      AppendParam(param, layer_id, param_id);
    }
    // Finally, set the backward flag
    layer_need_backward_.push_back(need_backward);
    if (need_backward) {
      for (int top_id = 0; top_id < top_id_vecs_[layer_id].size(); ++top_id) {
        blob_need_backward_[top_id_vecs_[layer_id][top_id]] = true;
      }
    }
  }
  // Go through the net backwards to determine which blobs contribute to the
  // loss.  We can skip backward computation for blobs that don't contribute
  // to the loss.
  // Also checks if all bottom blobs don't need backward computation (possible
  // because the skip_propagate_down param) and so we can skip bacward
  // computation for the entire layer
  set<string> blobs_under_loss;
  set<string> blobs_skip_backp;
  for (int layer_id = layers_.size() - 1; layer_id >= 0; --layer_id) {
    bool layer_contributes_loss = false;
    bool layer_skip_propagate_down = true;
    for (int top_id = 0; top_id < top_vecs_[layer_id].size(); ++top_id) {
      const string& blob_name = blob_names_[top_id_vecs_[layer_id][top_id]];
      if (layers_[layer_id]->loss(top_id) ||
          (blobs_under_loss.find(blob_name) != blobs_under_loss.end())) {
        layer_contributes_loss = true;
      }
      if (blobs_skip_backp.find(blob_name) == blobs_skip_backp.end()) {
        layer_skip_propagate_down = false;
      }
      if (layer_contributes_loss && !layer_skip_propagate_down)
        break;
    }
    // If this layer can skip backward computation, also all his bottom blobs
    // don't need backpropagation
    if (layer_need_backward_[layer_id] && layer_skip_propagate_down) {
      layer_need_backward_[layer_id] = false;
      for (int bottom_id = 0; bottom_id < bottom_vecs_[layer_id].size();
               ++bottom_id) {
        bottom_need_backward_[layer_id][bottom_id] = false;
      }
    }
    if (!layer_contributes_loss) { layer_need_backward_[layer_id] = false; }
    if (Caffe::root_solver()) {
      if (layer_need_backward_[layer_id]) {
        LOG(INFO) << layer_names_[layer_id] << " needs backward computation.";
      } else {
        LOG(INFO) << layer_names_[layer_id]
            << " does not need backward computation.";
      }
    }
    for (int bottom_id = 0; bottom_id < bottom_vecs_[layer_id].size();
         ++bottom_id) {
      if (layer_contributes_loss) {
        const string& blob_name =
            blob_names_[bottom_id_vecs_[layer_id][bottom_id]];
        blobs_under_loss.insert(blob_name);
      } else {
        bottom_need_backward_[layer_id][bottom_id] = false;
      }
      if (!bottom_need_backward_[layer_id][bottom_id]) {
        const string& blob_name =
                   blob_names_[bottom_id_vecs_[layer_id][bottom_id]];
        blobs_skip_backp.insert(blob_name);
      }
    }
  }
  // Handle force_backward if needed.
  if (param.force_backward()) {
    for (int layer_id = 0; layer_id < layers_.size(); ++layer_id) {
      layer_need_backward_[layer_id] = true;
      for (int bottom_id = 0;
           bottom_id < bottom_need_backward_[layer_id].size(); ++bottom_id) {
        bottom_need_backward_[layer_id][bottom_id] =
            bottom_need_backward_[layer_id][bottom_id] ||
            layers_[layer_id]->AllowForceBackward(bottom_id);
        blob_need_backward_[bottom_id_vecs_[layer_id][bottom_id]] =
            blob_need_backward_[bottom_id_vecs_[layer_id][bottom_id]] ||
            bottom_need_backward_[layer_id][bottom_id];
      }
      for (int param_id = 0; param_id < layers_[layer_id]->blobs().size();
           ++param_id) {
        layers_[layer_id]->set_param_propagate_down(param_id, true);
      }
    }
  }
  // In the end, all remaining blobs are considered output blobs.(所有剩下的Blob都被看作输出Blob)
  for (set<string>::iterator it = available_blobs.begin();
      it != available_blobs.end(); ++it) {
    LOG_IF(INFO, Caffe::root_solver())
        << "This network produces output " << *it;
    net_output_blobs_.push_back(blobs_[blob_name_to_idx[*it]].get());
    net_output_blob_indices_.push_back(blob_name_to_idx[*it]);
  }
  //将Blob名称与Blob id对应关系登记到Net后台数据库
  for (size_t blob_id = 0; blob_id < blob_names_.size(); ++blob_id) {
    blob_names_index_[blob_names_[blob_id]] = blob_id;
  }
  //将Layer名称与Layer id对应关系登记到Net后台数据库
  for (size_t layer_id = 0; layer_id < layer_names_.size(); ++layer_id) {
    layer_names_index_[layer_names_[layer_id]] = layer_id;
  }
  ShareWeights();
  debug_info_ = param.debug_info();
  LOG_IF(INFO, Caffe::root_solver()) << "Network initialization done.";
}

到这里我们大概了解了一个Net初始化的过程,关于其中三个登记注册函数,后面继续学习.