Gentoo中文社区

void QuicCryptoClientStream::DoHandshakeLoop(const CryptoHandshakeMessage* in) {
  QuicCryptoClientConfig::CachedState* cached =
      crypto_config_->LookupOrCreate(server_id_);

  QuicAsyncStatus rv = QUIC_SUCCESS;
  do {
    CHECK_NE(STATE_NONE, next_state_);
    const State state = next_state_;
    next_state_ = STATE_IDLE;
    rv = QUIC_SUCCESS;
    switch (state) {
      case STATE_INITIALIZE:
        DoInitialize(cached);
        break;
      case STATE_SEND_CHLO:
        DoSendCHLO(cached);
        return;  // return waiting to hear from server.
      case STATE_RECV_REJ:
        DoReceiveREJ(in, cached);
        break;
      case STATE_VERIFY_PROOF:
        rv = DoVerifyProof(cached);
        break;
      case STATE_VERIFY_PROOF_COMPLETE:
        DoVerifyProofComplete(cached);
        break;
      case STATE_GET_CHANNEL_ID:
        rv = DoGetChannelID(cached);
        break;
      case STATE_GET_CHANNEL_ID_COMPLETE:
        DoGetChannelIDComplete();
        break;
      case STATE_RECV_SHLO:
        DoReceiveSHLO(in, cached);
        break;
      case STATE_IDLE:
        // This means that the peer sent us a message that we weren't expecting.
        CloseConnectionWithDetails(QUIC_INVALID_CRYPTO_MESSAGE_TYPE,
                                   "Handshake in idle state");
        return;
      case STATE_INITIALIZE_SCUP:
        DoInitializeServerConfigUpdate(cached);
        break;
      case STATE_NONE:
        NOTREACHED();
        return;  // We are done.
    }
  } while (rv != QUIC_PENDING && next_state_ != STATE_NONE);
}

/*发送client hello消息*/
void QuicCryptoClientStream::DoSendCHLO(
    QuicCryptoClientConfig::CachedState* cached) {
  if (stateless_reject_received_) {//如果收到了server拒绝的消息
    // If we've gotten to this point, we've sent at least one hello
    // and received a stateless reject in response.  We cannot
    // continue to send hellos because the server has abandoned state
    // for this connection.  Abandon further handshakes.
    next_state_ = STATE_NONE;
    if (session()->connection()->connected()) {
      session()->connection()->CloseConnection(//关闭连接
          QUIC_CRYPTO_HANDSHAKE_STATELESS_REJECT, "stateless reject received",
          ConnectionCloseBehavior::SILENT_CLOSE);
    }
    return;
  }

  // Send the client hello in plaintext.
  //注意：这是client hello消息，没必要加密
  session()->connection()->SetDefaultEncryptionLevel(ENCRYPTION_NONE);
  encryption_established_ = false;
  if (num_client_hellos_ > kMaxClientHellos) {//握手消息已经发送了很多，不能再发了
    CloseConnectionWithDetails(
        QUIC_CRYPTO_TOO_MANY_REJECTS,
        base::StringPrintf("More than %u rejects", kMaxClientHellos).c_str());
    return;
  }
  num_client_hellos_++;
  //开始构造握手消息了
  CryptoHandshakeMessage out;
  DCHECK(session() != nullptr);
  DCHECK(session()->config() != nullptr);
  // Send all the options, regardless of whether we're sending an
  // inchoate or subsequent hello.
  /*填充握手消息的各个字段*/
  session()->config()->ToHandshakeMessage(&out);

  // Send a local timestamp to the server.
  out.SetValue(kCTIM,
               session()->connection()->clock()->WallNow().ToUNIXSeconds());

  if (!cached->IsComplete(session()->connection()->clock()->WallNow())) {
    crypto_config_->FillInchoateClientHello(
        server_id_, session()->connection()->supported_versions().front(),
        cached, session()->connection()->random_generator(),
        /* demand_x509_proof= */ true, &crypto_negotiated_params_, &out);
    // Pad the inchoate client hello to fill up a packet.
    const QuicByteCount kFramingOverhead = 50;  // A rough estimate.
    const QuicByteCount max_packet_size =
        session()->connection()->max_packet_length();
    if (max_packet_size <= kFramingOverhead) {
      DLOG(DFATAL) << "max_packet_length (" << max_packet_size
                   << ") has no room for framing overhead.";
      CloseConnectionWithDetails(QUIC_INTERNAL_ERROR,
                                 "max_packet_size too smalll");
      return;
    }
    if (kClientHelloMinimumSize > max_packet_size - kFramingOverhead) {
      DLOG(DFATAL) << "Client hello won't fit in a single packet.";
      CloseConnectionWithDetails(QUIC_INTERNAL_ERROR, "CHLO too large");
      return;
    }
    // TODO(rch): Remove this when we remove:
    // FLAGS_quic_use_chlo_packet_size
    out.set_minimum_size(
        static_cast<size_t>(max_packet_size - kFramingOverhead));
    next_state_ = STATE_RECV_REJ;
    /*做hash签名，接收方会根据hash验证消息是否完整*/
    CryptoUtils::HashHandshakeMessage(out, &chlo_hash_);
    //发送消息
    SendHandshakeMessage(out);
    return;
  }

  // If the server nonce is empty, copy over the server nonce from a previous
  // SREJ, if there is one. 
  if (FLAGS_enable_quic_stateless_reject_support &&
      crypto_negotiated_params_.server_nonce.empty() &&
      cached->has_server_nonce()) {
    crypto_negotiated_params_.server_nonce = cached->GetNextServerNonce();
    DCHECK(!crypto_negotiated_params_.server_nonce.empty());
  }

  string error_details;
  /*继续填充client hello消息*/
  QuicErrorCode error = crypto_config_->FillClientHello(
      server_id_, session()->connection()->connection_id(),
      session()->connection()->version(),
      session()->connection()->supported_versions().front(), cached,
      session()->connection()->clock()->WallNow(),
      //这个随机数会被server用来计算生成对称加密的key
      session()->connection()->random_generator(), 
      channel_id_key_.get(),
      //保存了nonce、key、token相关信息；后续对称加密的方法是CTR，需要NONCE值
      &crypto_negotiated_params_,
      &out, &error_details);
  if (error != QUIC_NO_ERROR) {
    // Flush the cached config so that, if it's bad, the server has a
    // chance to send us another in the future.
    cached->InvalidateServerConfig();
    CloseConnectionWithDetails(error, error_details);
    return;
  }
  /*继续对消息做hash，便于server验证收到的消息是否完整*/
  CryptoUtils::HashHandshakeMessage(out, &chlo_hash_);
  channel_id_sent_ = (channel_id_key_.get() != nullptr);
  if (cached->proof_verify_details()) {
    proof_handler_->OnProofVerifyDetailsAvailable(
        *cached->proof_verify_details());
  }
  next_state_ = STATE_RECV_SHLO;
  SendHandshakeMessage(out);
  // Be prepared to decrypt with the new server write key.
  session()->connection()->SetAlternativeDecrypter(
      ENCRYPTION_INITIAL,
      crypto_negotiated_params_.initial_crypters.decrypter.release(),
      true /* latch once used */);
  // Send subsequent packets under encryption on the assumption that the
  // server will accept the handshake.
  session()->connection()->SetEncrypter(
      ENCRYPTION_INITIAL,
      crypto_negotiated_params_.initial_crypters.encrypter.release());
  session()->connection()->SetDefaultEncryptionLevel(ENCRYPTION_INITIAL);

  // TODO(ianswett): Merge ENCRYPTION_REESTABLISHED and
  // ENCRYPTION_FIRST_ESTABLSIHED
  encryption_established_ = true;
  session()->OnCryptoHandshakeEvent(QuicSession::ENCRYPTION_REESTABLISHED);
}

QuicErrorCode QuicCryptoServerConfig::ProcessClientHello(
    const ValidateClientHelloResultCallback::Result& validate_chlo_result,
    bool reject_only,
    QuicConnectionId connection_id,
    const IPAddress& server_ip,
    const IPEndPoint& client_address,
    QuicVersion version,
    const QuicVersionVector& supported_versions,
    bool use_stateless_rejects,
    QuicConnectionId server_designated_connection_id,
    const QuicClock* clock,
    QuicRandom* rand,//发送给client用于计算对称key
    QuicCompressedCertsCache* compressed_certs_cache,
    QuicCryptoNegotiatedParameters* params,
    QuicCryptoProof* crypto_proof,
    QuicByteCount total_framing_overhead,
    QuicByteCount chlo_packet_size,
    CryptoHandshakeMessage* out,
    DiversificationNonce* out_diversification_nonce,
    string* error_details) const {
  DCHECK(error_details);

  const CryptoHandshakeMessage& client_hello =
      validate_chlo_result.client_hello;
  const ClientHelloInfo& info = validate_chlo_result.info;

  QuicErrorCode valid = CryptoUtils::ValidateClientHello(
      client_hello, version, supported_versions, error_details);
  if (valid != QUIC_NO_ERROR)
    return valid;

  StringPiece requested_scid;
  client_hello.GetStringPiece(kSCID, &requested_scid);
  const QuicWallTime now(clock->WallNow());

  scoped_refptr<Config> requested_config;
  scoped_refptr<Config> primary_config;
  {
    base::AutoLock locked(configs_lock_);

    if (!primary_config_.get()) {
      *error_details = "No configurations loaded";
      return QUIC_CRYPTO_INTERNAL_ERROR;
    }

    if (!next_config_promotion_time_.IsZero() &&
        next_config_promotion_time_.IsAfter(now)) {
      SelectNewPrimaryConfig(now);
      DCHECK(primary_config_.get());
      DCHECK_EQ(configs_.find(primary_config_->id)->second, primary_config_);
    }

    // Use the config that the client requested in order to do key-agreement.
    // Otherwise give it a copy of |primary_config_| to use.
    primary_config = crypto_proof->config;
    requested_config = GetConfigWithScid(requested_scid);
  }

  if (validate_chlo_result.error_code != QUIC_NO_ERROR) {
    *error_details = validate_chlo_result.error_details;
    return validate_chlo_result.error_code;
  }

  out->Clear();

  if (!ClientDemandsX509Proof(client_hello)) {
    *error_details = "Missing or invalid PDMD";
    return QUIC_UNSUPPORTED_PROOF_DEMAND;
  }
  DCHECK(proof_source_.get());
  string chlo_hash;
  CryptoUtils::HashHandshakeMessage(client_hello, &chlo_hash);
  // No need to get a new proof if one was already generated.
  if (!crypto_proof->chain &&
      !proof_source_->GetProof(server_ip, info.sni.as_string(),
                               primary_config->serialized, version, chlo_hash,
                               &crypto_proof->chain, &crypto_proof->signature,
                               &crypto_proof->cert_sct)) {
    return QUIC_HANDSHAKE_FAILED;
  }

  StringPiece cert_sct;
  if (client_hello.GetStringPiece(kCertificateSCTTag, &cert_sct) &&
      cert_sct.empty()) {
    params->sct_supported_by_client = true;
  }

  if (!info.reject_reasons.empty() || !requested_config.get()) {
    BuildRejection(version, clock->WallNow(), *primary_config, client_hello,
                   info, validate_chlo_result.cached_network_params,
                   use_stateless_rejects, server_designated_connection_id, rand,
                   compressed_certs_cache, params, *crypto_proof,
                   total_framing_overhead, chlo_packet_size, out);
    return QUIC_NO_ERROR;
  }

  if (reject_only) {
    return QUIC_NO_ERROR;
  }

  const QuicTag* their_aeads;
  const QuicTag* their_key_exchanges;
  size_t num_their_aeads, num_their_key_exchanges;
  if (client_hello.GetTaglist(kAEAD, &their_aeads, &num_their_aeads) !=
          QUIC_NO_ERROR ||
      client_hello.GetTaglist(kKEXS, &their_key_exchanges,
                              &num_their_key_exchanges) != QUIC_NO_ERROR ||
      num_their_aeads != 1 || num_their_key_exchanges != 1) {
    *error_details = "Missing or invalid AEAD or KEXS";
    return QUIC_INVALID_CRYPTO_MESSAGE_PARAMETER;
  }

  size_t key_exchange_index;
  if (!QuicUtils::FindMutualTag(requested_config->aead, their_aeads,
                                num_their_aeads, QuicUtils::LOCAL_PRIORITY,
                                &params->aead, nullptr) ||
      !QuicUtils::FindMutualTag(requested_config->kexs, their_key_exchanges,
                                num_their_key_exchanges,
                                QuicUtils::LOCAL_PRIORITY,
                                &params->key_exchange, &key_exchange_index)) {
    *error_details = "Unsupported AEAD or KEXS";
    return QUIC_CRYPTO_NO_SUPPORT;
  }

  if (!requested_config->tb_key_params.empty()) {
    const QuicTag* their_tbkps;
    size_t num_their_tbkps;
    switch (client_hello.GetTaglist(kTBKP, &their_tbkps, &num_their_tbkps)) {
      case QUIC_CRYPTO_MESSAGE_PARAMETER_NOT_FOUND:
        break;
      case QUIC_NO_ERROR:
        if (QuicUtils::FindMutualTag(
                requested_config->tb_key_params, their_tbkps, num_their_tbkps,
                QuicUtils::LOCAL_PRIORITY, &params->token_binding_key_param,
                nullptr)) {
          break;
        }
      default:
        *error_details = "Invalid Token Binding key parameter";
        return QUIC_INVALID_CRYPTO_MESSAGE_PARAMETER;
    }
  }

  StringPiece public_value;
  /*提取client hello数据包发送的公钥，server要用来生成对称加密的key*/
  if (!client_hello.GetStringPiece(kPUBS, &public_value)) {
    *error_details = "Missing public value";
    return QUIC_INVALID_CRYPTO_MESSAGE_PARAMETER;
  }

  const KeyExchange* key_exchange =
      requested_config->key_exchanges[key_exchange_index];
  if (!key_exchange->CalculateSharedKey(public_value,
                                        &params->initial_premaster_secret)) {
    *error_details = "Invalid public value";
    return QUIC_INVALID_CRYPTO_MESSAGE_PARAMETER;
  }

  if (!info.sni.empty()) {
    std::unique_ptr<char[]> sni_tmp(new char[info.sni.length() + 1]);
    memcpy(sni_tmp.get(), info.sni.data(), info.sni.length());
    sni_tmp[info.sni.length()] = 0;
    params->sni = CryptoUtils::NormalizeHostname(sni_tmp.get());
  }

  string hkdf_suffix;
  //client hello消息序列化，便于提取？
  const QuicData& client_hello_serialized = client_hello.GetSerialized();
  /*根据一个原始密钥材料，用hkdf算法推导出指定长度的密钥；
    这里明显是要根据client hello的数据生成对称加密的密钥了
  */
  hkdf_suffix.reserve(sizeof(connection_id) + client_hello_serialized.length() +
                      requested_config->serialized.size());
  hkdf_suffix.append(reinterpret_cast<char*>(&connection_id),
                     sizeof(connection_id));
  hkdf_suffix.append(client_hello_serialized.data(),
                     client_hello_serialized.length());
  hkdf_suffix.append(requested_config->serialized);
  DCHECK(proof_source_.get());
  if (crypto_proof->chain->certs.empty()) {
    *error_details = "Failed to get certs";
    return QUIC_CRYPTO_INTERNAL_ERROR;
  }
  hkdf_suffix.append(crypto_proof->chain->certs.at(0));

  StringPiece cetv_ciphertext;
  if (requested_config->channel_id_enabled &&
      client_hello.GetStringPiece(kCETV, &cetv_ciphertext)) {
    CryptoHandshakeMessage client_hello_copy(client_hello);
    client_hello_copy.Erase(kCETV);
    client_hello_copy.Erase(kPAD);

    const QuicData& client_hello_copy_serialized =
        client_hello_copy.GetSerialized();
    string hkdf_input;
    hkdf_input.append(QuicCryptoConfig::kCETVLabel,
                      strlen(QuicCryptoConfig::kCETVLabel) + 1);
    hkdf_input.append(reinterpret_cast<char*>(&connection_id),
                      sizeof(connection_id));
    hkdf_input.append(client_hello_copy_serialized.data(),
                      client_hello_copy_serialized.length());
    hkdf_input.append(requested_config->serialized);

    CrypterPair crypters;
    if (!CryptoUtils::DeriveKeys(params->initial_premaster_secret, params->aead,
                                 info.client_nonce, info.server_nonce,
                                 hkdf_input, Perspective::IS_SERVER,
                                 CryptoUtils::Diversification::Never(),
                                 &crypters, nullptr /* subkey secret */)) {
      *error_details = "Symmetric key setup failed";
      return QUIC_CRYPTO_SYMMETRIC_KEY_SETUP_FAILED;
    }

    char plaintext[kMaxPacketSize];
    size_t plaintext_length = 0;
    const bool success = crypters.decrypter->DecryptPacket(
        kDefaultPathId, 0 /* packet number */,
        StringPiece() /* associated data */, cetv_ciphertext, plaintext,
        &plaintext_length, kMaxPacketSize);
    if (!success) {
      *error_details = "CETV decryption failure";
      return QUIC_INVALID_CRYPTO_MESSAGE_PARAMETER;
    }
    std::unique_ptr<CryptoHandshakeMessage> cetv(
        CryptoFramer::ParseMessage(StringPiece(plaintext, plaintext_length)));
    if (!cetv.get()) {
      *error_details = "CETV parse error";
      return QUIC_INVALID_CRYPTO_MESSAGE_PARAMETER;
    }

    StringPiece key, signature;
    if (cetv->GetStringPiece(kCIDK, &key) &&
        cetv->GetStringPiece(kCIDS, &signature)) {
      if (!ChannelIDVerifier::Verify(key, hkdf_input, signature)) {
        *error_details = "ChannelID signature failure";
        return QUIC_INVALID_CRYPTO_MESSAGE_PARAMETER;
      }

      params->channel_id = key.as_string();
    }
  }

  string hkdf_input;
  size_t label_len = strlen(QuicCryptoConfig::kInitialLabel) + 1;
  hkdf_input.reserve(label_len + hkdf_suffix.size());
  hkdf_input.append(QuicCryptoConfig::kInitialLabel, label_len);
  hkdf_input.append(hkdf_suffix);

  string* subkey_secret = &params->initial_subkey_secret;
  CryptoUtils::Diversification diversification =
      CryptoUtils::Diversification::Never();
  if (version > QUIC_VERSION_32) {
    rand->RandBytes(out_diversification_nonce->data(),
                    out_diversification_nonce->size());
    diversification =
        CryptoUtils::Diversification::Now(out_diversification_nonce);
  }

  if (!CryptoUtils::DeriveKeys(params->initial_premaster_secret, params->aead,
                               info.client_nonce, info.server_nonce, hkdf_input,
                               Perspective::IS_SERVER, diversification,
                               &params->initial_crypters, subkey_secret)) {
    *error_details = "Symmetric key setup failed";
    return QUIC_CRYPTO_SYMMETRIC_KEY_SETUP_FAILED;
  }

  string forward_secure_public_value;
  if (ephemeral_key_source_.get()) {
    params->forward_secure_premaster_secret =
        ephemeral_key_source_->CalculateForwardSecureKey(
            key_exchange, rand, clock->ApproximateNow(), public_value,
            &forward_secure_public_value);
  } else {
    std::unique_ptr<KeyExchange> forward_secure_key_exchange(
        key_exchange->NewKeyPair(rand));
    forward_secure_public_value =
        forward_secure_key_exchange->public_value().as_string();
    /*生成共享密钥*/
    if (!forward_secure_key_exchange->CalculateSharedKey(
            public_value, &params->forward_secure_premaster_secret)) {
      *error_details = "Invalid public value";
      return QUIC_INVALID_CRYPTO_MESSAGE_PARAMETER;
    }
  }

  string forward_secure_hkdf_input;
  label_len = strlen(QuicCryptoConfig::kForwardSecureLabel) + 1;
  forward_secure_hkdf_input.reserve(label_len + hkdf_suffix.size());
  forward_secure_hkdf_input.append(QuicCryptoConfig::kForwardSecureLabel,
                                   label_len);
  forward_secure_hkdf_input.append(hkdf_suffix);

  string shlo_nonce;
  shlo_nonce = NewServerNonce(rand, info.now);
  out->SetStringPiece(kServerNonceTag, shlo_nonce);
  /*生成密钥*/
  if (!CryptoUtils::DeriveKeys(
          params->forward_secure_premaster_secret, params->aead,
          info.client_nonce,
          shlo_nonce.empty() ? info.server_nonce : shlo_nonce,
          forward_secure_hkdf_input, Perspective::IS_SERVER,
          CryptoUtils::Diversification::Never(),
          &params->forward_secure_crypters, &params->subkey_secret)) {
    *error_details = "Symmetric key setup failed";
    return QUIC_CRYPTO_SYMMETRIC_KEY_SETUP_FAILED;
  }

  out->set_tag(kSHLO);
  QuicTagVector supported_version_tags;
  for (size_t i = 0; i < supported_versions.size(); ++i) {
    supported_version_tags.push_back(
        QuicVersionToQuicTag(supported_versions[i]));
  }
  out->SetVector(kVER, supported_version_tags);
  out->SetStringPiece(
      kSourceAddressTokenTag,
      NewSourceAddressToken(*requested_config.get(), info.source_address_tokens,
                            client_address.address(), rand, info.now, nullptr));
  QuicSocketAddressCoder address_coder(client_address);
  out->SetStringPiece(kCADR, address_coder.Encode());
  /*server hello包中设置server的公钥，后续client会利用这个生成对称加密的key*/
  out->SetStringPiece(kPUBS, forward_secure_public_value);

  return QUIC_NO_ERROR;
}

bool Curve25519KeyExchange::CalculateSharedKey(StringPiece peer_public_value,
                                               string* out_result) const {
  if (peer_public_value.size() != crypto::curve25519::kBytes) {
    return false;
  }

  uint8_t result[crypto::curve25519::kBytes];
  if (!crypto::curve25519::ScalarMult(
          private_key_,
          reinterpret_cast<const uint8_t*>(peer_public_value.data()), result)) {
    return false;
  }
  out_result->assign(reinterpret_cast<char*>(result), sizeof(result));

  return true;
}
bool ScalarMult(const uint8_t* private_key,
                const uint8_t* peer_public_key,
                uint8_t* shared_key) {
  return !!X25519(shared_key, private_key, peer_public_key);
}

bool AeadBaseEncrypter::EncryptPacket(QuicPathId path_id,
                                      QuicPacketNumber packet_number,
                                      StringPiece associated_data,
                                      StringPiece plaintext,
                                      char* output,
                                      size_t* output_length,
                                      size_t max_output_length) {
  size_t ciphertext_size = GetCiphertextSize(plaintext.length());
  if (max_output_length < ciphertext_size) {
    return false;
  }
  // TODO(ianswett): Introduce a check to ensure that we don't encrypt with the
  // same packet number twice.
  const size_t nonce_size = nonce_prefix_size_ + sizeof(packet_number);
  ALIGNAS(4) char nonce_buffer[kMaxNonceSize];
  memcpy(nonce_buffer, nonce_prefix_, nonce_prefix_size_);
  uint64_t path_id_packet_number =
      QuicUtils::PackPathIdAndPacketNumber(path_id, packet_number);
  memcpy(nonce_buffer + nonce_prefix_size_, &path_id_packet_number,
         sizeof(path_id_packet_number));
  /*这里用nonce给明文加密*/
  if (!Encrypt(StringPiece(nonce_buffer, nonce_size), associated_data,
               plaintext, reinterpret_cast<unsigned char*>(output))) {
    return false;
  }
  *output_length = ciphertext_size;
  return true;
}

void QuicCryptoServerStream::FinishProcessingHandshakeMessage(
    const ValidateClientHelloResultCallback::Result& result,
    std::unique_ptr<ProofSource::Details> details) {
  const CryptoHandshakeMessage& message = result.client_hello;

  // Clear the callback that got us here.
  DCHECK(validate_client_hello_cb_ != nullptr);
  validate_client_hello_cb_ = nullptr;

  if (use_stateless_rejects_if_peer_supported_) {
    peer_supports_stateless_rejects_ = DoesPeerSupportStatelessRejects(message);
  }

  CryptoHandshakeMessage reply;
  DiversificationNonce diversification_nonce;
  string error_details;
  QuicErrorCode error = 
  /*server处理client的hello消息：重点是生成对称加密key、自己的公钥和nonce
   同时生成给client回复的消息*/
      ProcessClientHello(result, std::move(details), &reply,
                         &diversification_nonce, &error_details);

  if (error != QUIC_NO_ERROR) {
    CloseConnectionWithDetails(error, error_details);
    return;
  }

  if (reply.tag() != kSHLO) {
    if (reply.tag() == kSREJ) {
      DCHECK(use_stateless_rejects_if_peer_supported_);
      DCHECK(peer_supports_stateless_rejects_);
      // Before sending the SREJ, cause the connection to save crypto packets
      // so that they can be added to the time wait list manager and
      // retransmitted.
      session()->connection()->EnableSavingCryptoPackets();
    }
    SendHandshakeMessage(reply);//给client发server hello

    if (reply.tag() == kSREJ) {
      DCHECK(use_stateless_rejects_if_peer_supported_);
      DCHECK(peer_supports_stateless_rejects_);
      DCHECK(!handshake_confirmed());
      DVLOG(1) << "Closing connection "
               << session()->connection()->connection_id()
               << " because of a stateless reject.";
      session()->connection()->CloseConnection(
          QUIC_CRYPTO_HANDSHAKE_STATELESS_REJECT, "stateless reject",
          ConnectionCloseBehavior::SILENT_CLOSE);
    }
    return;
  }

  // If we are returning a SHLO then we accepted the handshake.  Now
  // process the negotiated configuration options as part of the
  // session config.
  //代码到这里已经给client发送了client hello，表示server已经准备好接受数据了
  //这里保存一些双方协商好的通信配置
  QuicConfig* config = session()->config();
  OverrideQuicConfigDefaults(config);
  error = config->ProcessPeerHello(message, CLIENT, &error_details);
  if (error != QUIC_NO_ERROR) {
    CloseConnectionWithDetails(error, error_details);
    return;
  }

  session()->OnConfigNegotiated();

  config->ToHandshakeMessage(&reply);

  // Receiving a full CHLO implies the client is prepared to decrypt with
  // the new server write key.  We can start to encrypt with the new server
  // write key. 可以开始用服务端新生成的key解密数据了
  //
  // NOTE: the SHLO will be encrypted with the new server write key.
  /*既然在server已经生成了对称加密的key，这里可以用这个key加密server hello消息*/
  session()->connection()->SetEncrypter(
      ENCRYPTION_INITIAL,
      crypto_negotiated_params_.initial_crypters.encrypter.release());
  session()->connection()->SetDefaultEncryptionLevel(ENCRYPTION_INITIAL);
  // Set the decrypter immediately so that we no longer accept unencrypted
  // packets.
  session()->connection()->SetDecrypter(
      ENCRYPTION_INITIAL,
      crypto_negotiated_params_.initial_crypters.decrypter.release());
  if (version() > QUIC_VERSION_32) {
    session()->connection()->SetDiversificationNonce(diversification_nonce);
  }

  SendHandshakeMessage(reply);//发送server hello

  session()->connection()->SetEncrypter(
      ENCRYPTION_FORWARD_SECURE,
      crypto_negotiated_params_.forward_secure_crypters.encrypter.release());
  session()->connection()->SetDefaultEncryptionLevel(ENCRYPTION_FORWARD_SECURE);

  session()->connection()->SetAlternativeDecrypter(
      ENCRYPTION_FORWARD_SECURE,
      crypto_negotiated_params_.forward_secure_crypters.decrypter.release(),
      false /* don't latch */);

  encryption_established_ = true;
  handshake_confirmed_ = true;
  session()->OnCryptoHandshakeEvent(QuicSession::HANDSHAKE_CONFIRMED);
}

struct NET_EXPORT_PRIVATE QuicStreamFrame {
  QuicStreamFrame();
  QuicStreamFrame(QuicStreamId stream_id,
                  bool fin,
                  QuicStreamOffset offset,
                  base::StringPiece data);
  QuicStreamFrame(QuicStreamId stream_id,
                  bool fin,
                  QuicStreamOffset offset,
                  QuicPacketLength data_length,
                  UniqueStreamBuffer buffer);
  ~QuicStreamFrame();

  NET_EXPORT_PRIVATE friend std::ostream& operator<<(std::ostream& os,
                                                     const QuicStreamFrame& s);

  QuicStreamId stream_id;
  bool fin;
  QuicPacketLength data_length;
  const char* data_buffer;
  QuicStreamOffset offset;  // Location of this data in the stream.
  // nullptr when the QuicStreamFrame is received, and non-null when sent.
  UniqueStreamBuffer buffer;

 private:
  QuicStreamFrame(QuicStreamId stream_id,
                  bool fin,
                  QuicStreamOffset offset,
                  const char* data_buffer,
                  QuicPacketLength data_length,
                  UniqueStreamBuffer buffer);

  DISALLOW_COPY_AND_ASSIGN(QuicStreamFrame);
};

QuicErrorCode QuicStreamSequencerBuffer::OnStreamData(
    QuicStreamOffset starting_offset,
    base::StringPiece data,
    QuicTime timestamp,
    size_t* const bytes_buffered,
    std::string* error_details) {
  *bytes_buffered = 0;
  QuicStreamOffset offset = starting_offset;
  size_t size = data.size();
  if (size == 0) {
    *error_details = "Received empty stream frame without FIN.";
    return QUIC_EMPTY_STREAM_FRAME_NO_FIN;
  }

  // Find the first gap not ending before |offset|. This gap maybe the gap to
  // fill if the arriving frame doesn't overlaps with previous ones.
  std::list<Gap>::iterator current_gap = gaps_.begin();
  while (current_gap != gaps_.end() && current_gap->end_offset <= offset) {
    ++current_gap;
  }

  DCHECK(current_gap != gaps_.end());

  // "duplication": might duplicate with data alread filled,but also might
  // overlap across different base::StringPiece objects already written.
  // In both cases, don't write the data,
  // and allow the caller of this method to handle the result.
  if (offset < current_gap->begin_offset &&
      offset + size <= current_gap->begin_offset) {
    DVLOG(1) << "Duplicated data at offset: " << offset << " length: " << size;
    return QUIC_NO_ERROR;
  }
  if (offset < current_gap->begin_offset &&
      offset + size > current_gap->begin_offset) {
    // Beginning of new data overlaps data before current gap.
    *error_details =
        string("Beginning of received data overlaps with buffered data.\n") +
        "New frame range " + RangeDebugString(offset, offset + size) +
        " with first 128 bytes: " +
        string(data.data(), data.length() < 128 ? data.length() : 128) +
        "\nCurrently received frames: " + ReceivedFramesDebugString() +
        "\nCurrent gaps: " + GapsDebugString();
    return QUIC_OVERLAPPING_STREAM_DATA;
  }
  if (offset + size > current_gap->end_offset) {
    // End of new data overlaps with data after current gap.
    *error_details =
        string("End of received data overlaps with buffered data.\n") +
        "New frame range " + RangeDebugString(offset, offset + size) +
        " with first 128 bytes: " +
        string(data.data(), data.length() < 128 ? data.length() : 128) +
        "\nCurrently received frames: " + ReceivedFramesDebugString() +
        "\nCurrent gaps: " + GapsDebugString();
    return QUIC_OVERLAPPING_STREAM_DATA;
  }

  // Write beyond the current range this buffer is covering.
  if (offset + size > total_bytes_read_ + max_buffer_capacity_bytes_) {
    *error_details = "Received data beyond available range.";
    return QUIC_INTERNAL_ERROR;
  }

  if (current_gap->begin_offset != starting_offset &&
      current_gap->end_offset != starting_offset + data.length() &&
      gaps_.size() >= kMaxNumGapsAllowed) {
    // This frame is going to create one more gap which exceeds max number of
    // gaps allowed. Stop processing.
    *error_details = "Too many gaps created for this stream.";
    return QUIC_TOO_MANY_FRAME_GAPS;
  }

  size_t total_written = 0;
  size_t source_remaining = size;
  const char* source = data.data();
  // Write data block by block. If corresponding block has not created yet,
  // create it first.
  // Stop when all data are written or reaches the logical end of the buffer.
  while (source_remaining > 0) {
    const size_t write_block_num = GetBlockIndex(offset);
    const size_t write_block_offset = GetInBlockOffset(offset);
    DCHECK_GT(blocks_count_, write_block_num);

    size_t block_capacity = GetBlockCapacity(write_block_num);
    size_t bytes_avail = block_capacity - write_block_offset;

    // If this write meets the upper boundary of the buffer,
    // reduce the available free bytes.
    if (offset + bytes_avail > total_bytes_read_ + max_buffer_capacity_bytes_) {
      bytes_avail = total_bytes_read_ + max_buffer_capacity_bytes_ - offset;
    }

    if (reduce_sequencer_buffer_memory_life_time_ && blocks_ == nullptr) {
      blocks_.reset(new BufferBlock*[blocks_count_]());
      for (size_t i = 0; i < blocks_count_; ++i) {
        blocks_[i] = nullptr;
      }
    }

    if (blocks_[write_block_num] == nullptr) {
      // TODO(danzh): Investigate if using a freelist would improve performance.
      // Same as RetireBlock().
      blocks_[write_block_num] = new BufferBlock();
    }

    const size_t bytes_to_copy = min<size_t>(bytes_avail, source_remaining);
    char* dest = blocks_[write_block_num]->buffer + write_block_offset;
    DVLOG(1) << "Write at offset: " << offset << " length: " << bytes_to_copy;
    memcpy(dest, source, bytes_to_copy);
    source += bytes_to_copy;
    source_remaining -= bytes_to_copy;
    offset += bytes_to_copy;
    total_written += bytes_to_copy;
  }

  DCHECK_GT(total_written, 0u);
  *bytes_buffered = total_written;
  UpdateGapList(current_gap, starting_offset, total_written);

  frame_arrival_time_map_.insert(
      std::make_pair(starting_offset, FrameInfo(size, timestamp)));
  num_bytes_buffered_ += total_written;
  return QUIC_NO_ERROR;
}

bool QuicConnection::WritePacket(SerializedPacket* packet) {
  /*如果数据包号比最后一个发送包的号还小，说明顺序错了，直接关闭连接*/
  if (packet->packet_number <
      sent_packet_manager_->GetLargestSentPacket(packet->path_id)) {
    QUIC_BUG << "Attempt to write packet:" << packet->packet_number << " after:"
             << sent_packet_manager_->GetLargestSentPacket(packet->path_id);
    CloseConnection(QUIC_INTERNAL_ERROR, "Packet written out of order.",
                    ConnectionCloseBehavior::SEND_CONNECTION_CLOSE_PACKET);
    return true;
  }
  /*没有连接、没有加密的包是不能发的*/
  if (ShouldDiscardPacket(*packet)) {
    ++stats_.packets_discarded;
    return true;
  }
  .........................  
}

boolean enqueueMessage(Message msg, long when) {
        if (msg.target == null) {
            throw new IllegalArgumentException("Message must have a target.");
        }
        if (msg.isInUse()) {
            throw new IllegalStateException(msg + " This message is already in use.");
        }

        synchronized (this) {
            if (mQuitting) {
                IllegalStateException e = new IllegalStateException(
                        msg.target + " sending message to a Handler on a dead thread");
                Log.w(TAG, e.getMessage(), e);
                msg.recycle();
                return false;
            }
            msg.markInUse();
            msg.when = when;
            Message p = mMessages;
            boolean needWake;
            if (p == null || when == 0 || when < p.when) {
                // New head, wake up the event queue if blocked.
                msg.next = p;
                mMessages = msg;
                needWake = mBlocked;
            } else {
                // Inserted within the middle of the queue.  Usually we don't have to wake
                // up the event queue unless there is a barrier at the head of the queue
                // and the message is the earliest asynchronous message in the queue.
                needWake = mBlocked && p.target == null && msg.isAsynchronous();
                Message prev;
                for (;;) {
                    prev = p;
                    p = p.next;
                    /*这不是普通的队列，这是优先级队列，根据message的时间排序*/
                    if (p == null || when < p.when) {
                        break;
                    }
                    if (needWake && p.isAsynchronous()) {
                        needWake = false;
                    }
                }
                msg.next = p; // invariant: p == prev.next
                prev.next = msg;
            }

            // We can assume mPtr != 0 because mQuitting is false.
            if (needWake) {
                nativeWake(mPtr);
            }
        }
        return true;
    }

/*
    1、首先判断当前时间与链表头部的Message.when字段的大小，
        如果该Message可以处理，那么直接返回链表头部的Message，否则继续等待
    2、当设置了同步屏障之后，next函数将会忽略所有的同步消息，返回异步消息。
        换句话说就是，设置了同步屏障之后，Handler只会处理异步消息。再换句话说，
        同步屏障为Handler消息机制增加了一种简单的优先级机制，
        异步消息的优先级要高于同步消息2、
    */
    Message next() {
        // Return here if the message loop has already quit and been disposed.
        // This can happen if the application tries to restart a looper after quit
        // which is not supported.
        final long ptr = mPtr;
        if (ptr == 0) {
            return null;
        }

        int pendingIdleHandlerCount = -1; // -1 only during first iteration
        int nextPollTimeoutMillis = 0;
        for (;;) {
            if (nextPollTimeoutMillis != 0) {
                Binder.flushPendingCommands();
            }

            nativePollOnce(ptr, nextPollTimeoutMillis);

            synchronized (this) {
                // Try to retrieve the next message.  Return if found.
                final long now = SystemClock.uptimeMillis();
                Message prevMsg = null;
                Message msg = mMessages;
                if (msg != null && msg.target == null) {//碰到同步屏障
                    // Stalled by a barrier.  Find the next asynchronous message in the queue.
                    do {
                        //跳出循环时，msg指向离表头最近的一个异步消息
                        prevMsg = msg;
                        msg = msg.next;
                    } while (msg != null && !msg.isAsynchronous());
                }
                if (msg != null) {
                    if (now < msg.when) {
                        // Next message is not ready.  Set a timeout to wake up when it is ready.
                        // 如果下一个消息的处理时间还没到，那么设置一个等待超时时间
                        nextPollTimeoutMillis = (int) Math.min(msg.when - now, Integer.MAX_VALUE);
                    } else {
                        //拿到可用的msg，返回
                        // Got a message.
                        mBlocked = false;
                        if (prevMsg != null) {
                            prevMsg.next = msg.next;
                        } else {
                        //链表头部移动到链表第二个元素上
                            mMessages = msg.next;
                        }
                        msg.next = null;
                        if (DEBUG) Log.v(TAG, "Returning message: " + msg);
                        msg.markInUse();
                        //返回链表中的第一个元素
                        return msg;
                    }
                } else {
                    // No more messages.
                    nextPollTimeoutMillis = -1;
                }

                // Process the quit message now that all pending messages have been handled.
                if (mQuitting) {
                    dispose();
                    return null;
                }

                // If first time idle, then get the number of idlers to run.
                // Idle handles only run if the queue is empty or if the first message
                // in the queue (possibly a barrier) is due to be handled in the future.
                if (pendingIdleHandlerCount < 0
                        && (mMessages == null || now < mMessages.when)) {
                    pendingIdleHandlerCount = mIdleHandlers.size();
                }
                if (pendingIdleHandlerCount <= 0) {
                    // No idle handlers to run.  Loop and wait some more.
                    mBlocked = true;
                    continue;
                }

                if (mPendingIdleHandlers == null) {
                    mPendingIdleHandlers = new IdleHandler[Math.max(pendingIdleHandlerCount, 4)];
                }
                mPendingIdleHandlers = mIdleHandlers.toArray(mPendingIdleHandlers);
            }

            // Run the idle handlers.
            // We only ever reach this code block during the first iteration.
            for (int i = 0; i < pendingIdleHandlerCount; i++) {
                final IdleHandler idler = mPendingIdleHandlers[i];
                mPendingIdleHandlers[i] = null; // release the reference to the handler

                boolean keep = false;
                try {
                    keep = idler.queueIdle();
                } catch (Throwable t) {
                    Log.wtf(TAG, "IdleHandler threw exception", t);
                }

                if (!keep) {
                    synchronized (this) {
                        mIdleHandlers.remove(idler);
                    }
                }
            }

            // Reset the idle handler count to 0 so we do not run them again.
            pendingIdleHandlerCount = 0;

            // While calling an idle handler, a new message could have been delivered
            // so go back and look again for a pending message without waiting.
            nextPollTimeoutMillis = 0;
        }
    }

/**
     * Run the message queue in this thread. Be sure to call
     * {@link #quit()} to end the loop.
     让MessageQueue循环动起来：通过next筛选出超时的消息，然后通过dispatchMessage让消费者线程处理
     */
    public static void loop() {
        final Looper me = myLooper();
        if (me == null) {
            throw new RuntimeException("No Looper; Looper.prepare() wasn't called on this thread.");
        }
        final MessageQueue queue = me.mQueue;

        // Make sure the identity of this thread is that of the local process,
        // and keep track of what that identity token actually is.
        Binder.clearCallingIdentity();
        final long ident = Binder.clearCallingIdentity();

        // Allow overriding a threshold with a system prop. e.g.
        // adb shell 'setprop log.looper.1000.main.slow 1 && stop && start'
        final int thresholdOverride =
                SystemProperties.getInt("log.looper."
                        + Process.myUid() + "."
                        + Thread.currentThread().getName()
                        + ".slow", 0);

        boolean slowDeliveryDetected = false;

        for (;;) {
            //获取消息队列超时的消息
            Message msg = queue.next(); // might block
            if (msg == null) {//没有超时的消息先返回了
                // No message indicates that the message queue is quitting.
                return;
            }

            // This must be in a local variable, in case a UI event sets the logger
            final Printer logging = me.mLogging;
            if (logging != null) {
                logging.println(">>>>> Dispatching to " + msg.target + " " +
                        msg.callback + ": " + msg.what);
            }
            // Make sure the observer won't change while processing a transaction.
            final Observer observer = sObserver;

            final long traceTag = me.mTraceTag;
            long slowDispatchThresholdMs = me.mSlowDispatchThresholdMs;
            long slowDeliveryThresholdMs = me.mSlowDeliveryThresholdMs;
            if (thresholdOverride > 0) {
                slowDispatchThresholdMs = thresholdOverride;
                slowDeliveryThresholdMs = thresholdOverride;
            }
            final boolean logSlowDelivery = (slowDeliveryThresholdMs > 0) && (msg.when > 0);
            final boolean logSlowDispatch = (slowDispatchThresholdMs > 0);

            final boolean needStartTime = logSlowDelivery || logSlowDispatch;
            final boolean needEndTime = logSlowDispatch;

            if (traceTag != 0 && Trace.isTagEnabled(traceTag)) {
                Trace.traceBegin(traceTag, msg.target.getTraceName(msg));
            }

            final long dispatchStart = needStartTime ? SystemClock.uptimeMillis() : 0;
            final long dispatchEnd;
            Object token = null;
            if (observer != null) {
                token = observer.messageDispatchStarting();
            }
            long origWorkSource = ThreadLocalWorkSource.setUid(msg.workSourceUid);
            try {
                /*
                msg.target是一个Handler，让该Message关联的Handler通过dispatchMessage()处理Message。
                */
                msg.target.dispatchMessage(msg);
                if (observer != null) {
                    observer.messageDispatched(token, msg);
                }
                dispatchEnd = needEndTime ? SystemClock.uptimeMillis() : 0;
            } catch (Exception exception) {
                if (observer != null) {
                    observer.dispatchingThrewException(token, msg, exception);
                }
                throw exception;
            } finally {
                ThreadLocalWorkSource.restore(origWorkSource);
                if (traceTag != 0) {
                    Trace.traceEnd(traceTag);
                }
            }
            if (logSlowDelivery) {
                if (slowDeliveryDetected) {
                    if ((dispatchStart - msg.when) <= 10) {
                        Slog.w(TAG, "Drained");
                        slowDeliveryDetected = false;
                    }
                } else {
                    if (showSlowLog(slowDeliveryThresholdMs, msg.when, dispatchStart, "delivery",
                            msg)) {
                        // Once we write a slow delivery log, suppress until the queue drains.
                        slowDeliveryDetected = true;
                    }
                }
            }
            if (logSlowDispatch) {
                showSlowLog(slowDispatchThresholdMs, dispatchStart, dispatchEnd, "dispatch", msg);
            }

            if (logging != null) {
                logging.println("<<<<< Finished to " + msg.target + " " + msg.callback);
            }

            // Make sure that during the course of dispatching the
            // identity of the thread wasn't corrupted.
            final long newIdent = Binder.clearCallingIdentity();
            if (ident != newIdent) {
                Log.wtf(TAG, "Thread identity changed from 0x"
                        + Long.toHexString(ident) + " to 0x"
                        + Long.toHexString(newIdent) + " while dispatching to "
                        + msg.target.getClass().getName() + " "
                        + msg.callback + " what=" + msg.what);
            }

            msg.recycleUnchecked();
        }
    }

/**
     * Handle system messages here.
     */
    public void dispatchMessage(@NonNull Message msg) {
        //消息的回调函数不会空就执行回调函数
        if (msg.callback != null) {
            handleCallback(msg);
        } else {
            if (mCallback != null) {
                if (mCallback.handleMessage(msg)) {
                    return;
                }
            }
            //用户实现的接口方法
            handleMessage(msg);
        }
    }

/**
     * Return a new Message instance from the global pool. Allows us to
     * avoid allocating new objects in many cases.
     */
    public static Message obtain() {
        synchronized (sPoolSync) {
            if (sPool != null) {
                Message m = sPool;
                sPool = m.next;
                m.next = null;
                m.flags = 0; // clear in-use flag
                sPoolSize--;
                return m;
            }
        }
        return new Message();
    }

import time

def consumer():
    r = ''
    while True:
        n = yield r
        if not n:
            return
        print('[CONSUMER] Consuming %s...' % n)
        time.sleep(1)
        r = '200 OK'

def produce(c):
    c.next()
    n = 0
    while n < 5:
        n = n + 1
        print('[PRODUCER] Producing %s...' % n)
        r = c.send(n)
        print('[PRODUCER] Consumer return: %s' % r)
    c.close()

if __name__=='__main__':
    c = consumer()
    produce(c)

while(1) {
    socklen_t len = sizeof(struct sockaddr_in);
    int clientfd = accept(sockfd, (struct sockaddr*)&remote, &len);

    pthread_t thread_id;
    pthread_create(&thread_id, NULL, client_cb, &clientfd);
}

while (1) {
    int nready = epoll_wait(epfd, events, EVENT_SIZE, -1);

    for (i = 0;i < nready;i ++) {

        int sockfd = events[i].data.fd;
        if (sockfd == listenfd) {
            int connfd = accept(listenfd, xxx, xxxx);
            
            setnonblock(connfd);

            ev.events = EPOLLIN | EPOLLET;
            ev.data.fd = connfd;
            epoll_ctl(epfd, EPOLL_CTL_ADD, connfd, &ev);

        } else {
            handle(sockfd);
        }
    }
}
int thread_cb(int sockfd) {
    // 此函数是在线程池创建的线程中运行。
    // 与handle不在一个线程上下文中运行
    recv(sockfd, rbuffer, length, 0);
    parser_proto(rbuffer, length);
    send(sockfd, sbuffer, length, 0);
}

int handle(int sockfd) {
    //此函数在主线程 main_thread 中运行
    //在此处之前，确保线程池已经启动。
    push_thread(sockfd, thread_cb); //将sockfd放到其他线程中运行（这里单独生成了线程池）。
}

#include<stdlib.h>
#include<sys/mman.h>
#include<fcntl.h>

int main(void)
{
    int *p;
    int fd = open("hello", O_RDWR);
    if(fd < 0)
    {
        perror("open hello");
        exit(1);
    }
    p = mmap(NULL,6, PROT_WRITE, MAP_SHARED, fd, 0);
    if(p == MAP_FAILED)
    {
       perror("mmap"); //程序进里面了，证明mmap失败
       exit(1);
    }
    close(fd);
    p[0] = 0x30313233;
    munmap(p, 6);
    return 0;

}

/*
 * The caller must hold down_write(&current->mm->mmap_sem).
 根据用户传入的参数做了一系列的检查，然后根据参数初始化vm_area_struct的标志vm_flags、
 vma->vm_file = get_file(file)建立文件与vma的映射
 */
unsigned long do_mmap(struct file *file, unsigned long addr,
            unsigned long len, unsigned long prot,
            unsigned long flags, vm_flags_t vm_flags,
            unsigned long pgoff, unsigned long *populate)
{
    struct mm_struct *mm = current->mm;//当前进程的虚拟内存描述符
    int pkey = 0;

    *populate = 0;

    if (!len)
        return -EINVAL;

    /*
     * Does the application expect PROT_READ to imply PROT_EXEC?
     *
     * (the exception is when the underlying filesystem is noexec
     *  mounted, in which case we dont add PROT_EXEC.)
     */
    if ((prot & PROT_READ) && (current->personality & READ_IMPLIES_EXEC))
        if (!(file && path_noexec(&file->f_path)))
            prot |= PROT_EXEC;
    /* 假如没有设置MAP_FIXED标志，且addr小于mmap_min_addr, 因为可以修改addr,
    所以就需要将addr设为mmap_min_addr的页对齐后的地址 */
    if (!(flags & MAP_FIXED))
        addr = round_hint_to_min(addr);

    /* Careful about overflows.. 检查长度，防止溢出??*/
    /* 进行Page大小的对齐,因为内存映射大小必须页对齐 */
    len = PAGE_ALIGN(len);
    if (!len)
        return -ENOMEM;

    /* offset overflow? */
    if ((pgoff + (len >> PAGE_SHIFT)) < pgoff)
        return -EOVERFLOW;

    /* Too many mappings? */
    /* 判断该进程的地址空间的虚拟区间数量是否超过了限制 */
    if (mm->map_count > sysctl_max_map_count)
        return -ENOMEM;

    /* Obtain the address to map to. we verify (or select) it and ensure
     * that it represents a valid section of the address space.
     从当前进程的用户空间获取一个未被映射区间的起始地址:这里就涉及到红黑树了 
     */
    addr = get_unmapped_area(file, addr, len, pgoff, flags);
    if (offset_in_page(addr))/* 检查addr是否有效 */
        return addr;

    if (prot == PROT_EXEC) {
        pkey = execute_only_pkey(mm);
        if (pkey < 0)
            pkey = 0;
    }

    /* Do simple checking here so the lower-level routines won't have
     * to. we assume access permissions have been handled by the open
     * of the memory object, so we don't do any here.
     */
    vm_flags |= calc_vm_prot_bits(prot, pkey) | calc_vm_flag_bits(flags) |
            mm->def_flags | VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC;
    /* 假如flags设置MAP_LOCKED，即类似于mlock()将申请的地址空间锁定在内存中, 
    检查是否可以进行lock*/
    if (flags & MAP_LOCKED)
        if (!can_do_mlock())
            return -EPERM;

    if (mlock_future_check(mm, vm_flags, len))
        return -EAGAIN;

    if (file) {
        struct inode *inode = file_inode(file);
        /*根据标志指定的map种类，把为文件设置的访问权考虑进去。
        如果所请求的内存映射是共享可写的，就要检查要映射的文件是为写入而打开的，而不
        是以追加模式打开的，还要检查文件上没有上强制锁。
        对于任何种类的内存映射，都要检查文件是否为读操作而打开的。
        */

        switch (flags & MAP_TYPE) {
        case MAP_SHARED:
            if ((prot&PROT_WRITE) && !(file->f_mode&FMODE_WRITE))
                return -EACCES;

            /*
             * Make sure we don't allow writing to an append-only
             * file..
             */
            if (IS_APPEND(inode) && (file->f_mode & FMODE_WRITE))
                return -EACCES;

            /*
             * Make sure there are no mandatory locks on the file.
             */
            if (locks_verify_locked(file))
                return -EAGAIN;

            vm_flags |= VM_SHARED | VM_MAYSHARE;
            if (!(file->f_mode & FMODE_WRITE))
                vm_flags &= ~(VM_MAYWRITE | VM_SHARED);

            /* fall through */
        case MAP_PRIVATE:
            if (!(file->f_mode & FMODE_READ))
                return -EACCES;
            if (path_noexec(&file->f_path)) {
                if (vm_flags & VM_EXEC)
                    return -EPERM;
                vm_flags &= ~VM_MAYEXEC;
            }

            if (!file->f_op->mmap)
                return -ENODEV;
            if (vm_flags & (VM_GROWSDOWN|VM_GROWSUP))
                return -EINVAL;
            break;

        default:
            return -EINVAL;
        }
    } else {
        switch (flags & MAP_TYPE) {
        case MAP_SHARED:
            if (vm_flags & (VM_GROWSDOWN|VM_GROWSUP))
                return -EINVAL;
            /*
             * Ignore pgoff.
             */
            pgoff = 0;
            vm_flags |= VM_SHARED | VM_MAYSHARE;
            break;
        case MAP_PRIVATE:
            /*
             * Set pgoff according to addr for anon_vma.
             */
            pgoff = addr >> PAGE_SHIFT;
            break;
        default:
            return -EINVAL;
        }
    }

    /*
     * Set 'VM_NORESERVE' if we should not account for the
     * memory use of this mapping.
     */
    if (flags & MAP_NORESERVE) {
        /* We honor MAP_NORESERVE if allowed to overcommit */
        if (sysctl_overcommit_memory != OVERCOMMIT_NEVER)
            vm_flags |= VM_NORESERVE;

        /* hugetlb applies strict overcommit unless MAP_NORESERVE */
        if (file && is_file_hugepages(file))
            vm_flags |= VM_NORESERVE;
    }
    /*创建和初始化虚拟内存区域，并加入红黑树管理*/
    addr = mmap_region(file, addr, len, vm_flags, pgoff);
    if (!IS_ERR_VALUE(addr) &&
        ((vm_flags & VM_LOCKED) ||
        /*
        假如没有设置MAP_POPULATE标志位内核并不在调用mmap()时就为进程分配物理内存空间，
        而是直到下次真正访问地址空间时发现数据不存在于物理内存空间时才触发Page Fault
        ，将缺失的 Page 换入内存空间
        */
         (flags & (MAP_POPULATE | MAP_NONBLOCK)) == MAP_POPULATE))
        *populate = len;
    return addr;
}

/*创建和初始化虚拟内存区域，并加入红黑树节点进行管理*/
unsigned long mmap_region(struct file *file, unsigned long addr,
        unsigned long len, vm_flags_t vm_flags, unsigned long pgoff)
{
    struct mm_struct *mm = current->mm;
    struct vm_area_struct *vma, *prev;
    int error;
    struct rb_node **rb_link, *rb_parent;
    unsigned long charged = 0;

    /* Check against address space limit.
       申请的虚拟内存空间是否超过了限制 */
    if (!may_expand_vm(mm, vm_flags, len >> PAGE_SHIFT)) {
        unsigned long nr_pages;

        /*
         * MAP_FIXED may remove pages of mappings that intersects with
         * requested mapping. Account for the pages it would unmap.
         */
        nr_pages = count_vma_pages_range(mm, addr, addr + len);

        if (!may_expand_vm(mm, vm_flags,
                    (len >> PAGE_SHIFT) - nr_pages))
            return -ENOMEM;
    }

    /* Clear old maps 
    检查[addr, addr+len)的区间是否存在映射空间，假如存在重合的映射空间需要munmap*/
    while (find_vma_links(mm, addr, addr + len, &prev, &rb_link,
                  &rb_parent)) {
        if (do_munmap(mm, addr, len))
            return -ENOMEM;
    }

    /*
     * Private writable mapping: check memory availability
     */
    if (accountable_mapping(file, vm_flags)) {
        charged = len >> PAGE_SHIFT;
        if (security_vm_enough_memory_mm(mm, charged))
            return -ENOMEM;
        vm_flags |= VM_ACCOUNT;
    }

    /*
     * Can we just expand an old mapping?
     检查是否可以合并[addr, addr+len)区间内的虚拟地址空间vma
     */
    vma = vma_merge(mm, prev, addr, addr + len, vm_flags,
            NULL, file, pgoff, NULL, NULL_VM_UFFD_CTX);
    if (vma)/* 假如合并成功，即使用合并后的vma, 并跳转至out */
        goto out;

    /*
     * Determine the object being mapped and call the appropriate
     * specific mapper. the address has already been validated, but
     * not unmapped, but the maps are removed from the list.
     如果不能和已有的虚拟内存区域合并，通过 Memory Descriptor 来申请一个 vma
     */
    vma = kmem_cache_zalloc(vm_area_cachep, GFP_KERNEL);
    if (!vma) {
        error = -ENOMEM;
        goto unacct_error;
    }

    vma->vm_mm = mm;
    vma->vm_start = addr;
    vma->vm_end = addr + len;
    vma->vm_flags = vm_flags;
    vma->vm_page_prot = vm_get_page_prot(vm_flags);
    vma->vm_pgoff = pgoff;
    INIT_LIST_HEAD(&vma->anon_vma_chain);//vma通过链表连接，这里初始化链表头
    
    /* 假如指定了文件映射 */
    if (file) {
         /* 映射的文件不允许写入，调用 deny_write_accsess(file) 排斥常规的文件操作 */
        if (vm_flags & VM_DENYWRITE) {
            error = deny_write_access(file);
            if (error)
                goto free_vma;
        }
        if (vm_flags & VM_SHARED) {/* 映射的文件允许其他进程可见, 标记文件为可写 */
            error = mapping_map_writable(file->f_mapping);
            if (error)
                goto allow_write_and_free_vma;
        }

        /* ->mmap() can change vma->vm_file, but must guarantee that
         * vma_link() below can deny write-access if VM_DENYWRITE is set
         * and map writably if VM_SHARED is set. This usually means the
         * new file must not have been exposed to user-space, yet.
         */
        vma->vm_file = get_file(file);/* 递增 File 的引用次数，返回 File 赋给 vma */
        error = file->f_op->mmap(file, vma); /* 调用文件系统指定的 mmap 函数*/
        if (error)
            goto unmap_and_free_vma;

        /* Can addr have changed??
         *
         * Answer: Yes, several device drivers can do it in their
         *         f_op->mmap method. -DaveM
         * Bug: If addr is changed, prev, rb_link, rb_parent should
         *      be updated for vma_link()
         */
        WARN_ON_ONCE(addr != vma->vm_start);

        addr = vma->vm_start;
        vm_flags = vma->vm_flags;
        /* 假如标志为 VM_SHARED，但没有指定映射文件，需要调用 shmem_zero_setup()
           shmem_zero_setup() 实际映射的文件是 dev/zero
        */
    } else if (vm_flags & VM_SHARED) {
        error = shmem_zero_setup(vma);
        if (error)
            goto free_vma;
    }
    /*新分配的vma加入红黑树*/
    vma_link(mm, vma, prev, rb_link, rb_parent);
    /* Once vma denies write, undo our temporary denial count */
    if (file) {
        if (vm_flags & VM_SHARED)
            mapping_unmap_writable(file->f_mapping);
        if (vm_flags & VM_DENYWRITE)
            allow_write_access(file);
    }
    file = vma->vm_file;
out:
    perf_event_mmap(vma);
    /* 更新进程的虚拟地址空间 mm */
    vm_stat_account(mm, vm_flags, len >> PAGE_SHIFT);
    if (vm_flags & VM_LOCKED) {
        if (!((vm_flags & VM_SPECIAL) || is_vm_hugetlb_page(vma) ||
                    vma == get_gate_vma(current->mm)))
            mm->locked_vm += (len >> PAGE_SHIFT);
        else
            vma->vm_flags &= VM_LOCKED_CLEAR_MASK;
    }

    if (file)
        uprobe_mmap(vma);

    /*
     * New (or expanded) vma always get soft dirty status.
     * Otherwise user-space soft-dirty page tracker won't
     * be able to distinguish situation when vma area unmapped,
     * then new mapped in-place (which must be aimed as
     * a completely new data area).
     */
    vma->vm_flags |= VM_SOFTDIRTY;

    vma_set_page_prot(vma);

    return addr;

unmap_and_free_vma:
    vma->vm_file = NULL;
    fput(file);

    /* Undo any partial mapping done by a device driver. */
    unmap_region(mm, vma, prev, vma->vm_start, vma->vm_end);
    charged = 0;
    if (vm_flags & VM_SHARED)
        mapping_unmap_writable(file->f_mapping);
allow_write_and_free_vma:
    if (vm_flags & VM_DENYWRITE)
        allow_write_access(file);
free_vma:
    kmem_cache_free(vm_area_cachep, vma);
unacct_error:
    if (charged)
        vm_unacct_memory(charged);
    return error;
}

/**
 * remap_pfn_range - remap kernel memory to userspace
   将内核空间的内存映射到用户空间，或者说是
   将用户空间的一个vma虚拟内存区映射到以page开始的一段连续物理页面上
 * @vma: user vma to map to：需要映射(或者说挂载关联)物理地址的vma
 * @addr: target user address to start at：用户空间地址的起始位置
 * @pfn: physical address of kernel memory:内核的物理地址空间
 * @size: size of map area
 * @prot: page protection flags for this mapping：内存页面的属性
 *
 *  Note: this is only safe if the mm semaphore is held when called.
 */
int remap_pfn_range(struct vm_area_struct *vma, unsigned long addr,
            unsigned long pfn, unsigned long size, pgprot_t prot)
{
    pgd_t *pgd;
    unsigned long next;
    /*需要映射的虚拟地址尾部：注意要页对齐，因为cpu硬件是以页为单位管理内存的*/
    unsigned long end = addr + PAGE_ALIGN(size);
    struct mm_struct *mm = vma->vm_mm;
    unsigned long remap_pfn = pfn;
    int err;

    /*
     * Physically remapped pages are special. Tell the
     * rest of the world about it:
     *   VM_IO tells people not to look at these pages
     *    (accesses can have side effects).
     *   VM_PFNMAP tells the core MM that the base pages are just
     *    raw PFN mappings, and do not have a "struct page" associated
     *    with them.
     *   VM_DONTEXPAND
     *      Disable vma merging and expanding with mremap().
     *   VM_DONTDUMP
     *      Omit vma from core dump, even when VM_IO turned off.
     *
     * There's a horrible special case to handle copy-on-write
     * behaviour that some programs depend on. We mark the "original"
     * un-COW'ed pages by matching them up with "vma->vm_pgoff".
     * See vm_normal_page() for details.
     */
    if (is_cow_mapping(vma->vm_flags)) {
        if (addr != vma->vm_start || end != vma->vm_end)
            return -EINVAL;
        vma->vm_pgoff = pfn;
    }

    err = track_pfn_remap(vma, &prot, remap_pfn, addr, PAGE_ALIGN(size));
    if (err)
        return -EINVAL;
    /*改变虚拟地址的标志*/
    vma->vm_flags |= VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP;

    BUG_ON(addr >= end);
    pfn -= addr >> PAGE_SHIFT;
    /*
    /* To find an entry in a generic PGD。宏定义展开后如下：
    #define pgd_index(address) (((address) >> PGDIR_SHIFT) & (PTRS_PER_PGD-1))
    #define pgd_offset(mm, address) ((mm)->pgd+pgd_index(address))
    查找addr第1级页目录项中对应的页表项的地址
    */
    pgd = pgd_offset(mm, addr);
    /*刷新TLB缓存；这个缓存和CPU的L1、L2、L3的缓存思想一致，
    既然进行地址转换需要的内存IO次数多，且耗时，
    那么干脆就在CPU里把页表尽可能地cache起来不就行了么，
    所以就有了TLB(Translation Lookaside Buffer)，
    专门用于改进虚拟地址到物理地址转换速度的缓存。
    其访问速度非常快，和寄存器相当，比L1访问还快。*/
    flush_cache_range(vma, addr, end);
    do {
        /*
        计算下一个将要被映射的虚拟地址，如果addr到end可以被一个pgd映射的话，那么返回end的值
        */
        next = pgd_addr_end(addr, end);
        /*完成虚拟内存和物理内存映射，本质就是填写完CR3指向的页表；
        过程就是逐级完成：1级是pgd，上面已经完成；2级是pud，3级是pmd，4级是pte
        */
        err = remap_pud_range(mm, pgd, addr, next,
                pfn + (addr >> PAGE_SHIFT), prot);
        if (err)
            break;
    } while (pgd++, addr = next, addr != end);

    if (err)
        untrack_pfn(vma, remap_pfn, PAGE_ALIGN(size));

    return err;
}

/*
 * maps a range of physical memory into the requested pages. the old
 * mappings are removed. any references to nonexistent pages results
 * in null mappings (currently treated as "copy-on-access")
 */
static int remap_pte_range(struct mm_struct *mm, pmd_t *pmd,
            unsigned long addr, unsigned long end,
            unsigned long pfn, pgprot_t prot)
{
    pte_t *pte;
    spinlock_t *ptl;

    pte = pte_alloc_map_lock(mm, pmd, addr, &ptl);
    if (!pte)
        return -ENOMEM;
    arch_enter_lazy_mmu_mode();
    do {
        BUG_ON(!pte_none(*pte));
        /*这是映射的最后一级：把物理地址的值填写到pte表项*/
        set_pte_at(mm, addr, pte, pte_mkspecial(pfn_pte(pfn, prot)));
        pfn++;
    } while (pte++, addr += PAGE_SIZE, addr != end);
    arch_leave_lazy_mmu_mode();
    pte_unmap_unlock(pte - 1, ptl);
    return 0;
}

static inline int remap_pmd_range(struct mm_struct *mm, pud_t *pud,
            unsigned long addr, unsigned long end,
            unsigned long pfn, pgprot_t prot)
{
    pmd_t *pmd;
    unsigned long next;

    pfn -= addr >> PAGE_SHIFT;
    pmd = pmd_alloc(mm, pud, addr);
    if (!pmd)
        return -ENOMEM;
    VM_BUG_ON(pmd_trans_huge(*pmd));
    do {
        next = pmd_addr_end(addr, end);
        if (remap_pte_range(mm, pmd, addr, next,
                pfn + (addr >> PAGE_SHIFT), prot))
            return -ENOMEM;
    } while (pmd++, addr = next, addr != end);
    return 0;
}

static inline int remap_pud_range(struct mm_struct *mm, pgd_t *pgd,
            unsigned long addr, unsigned long end,
            unsigned long pfn, pgprot_t prot)
{
    pud_t *pud;
    unsigned long next;

    pfn -= addr >> PAGE_SHIFT;
    /*返回pgd值*/
    pud = pud_alloc(mm, pgd, addr);
    if (!pud)
        return -ENOMEM;
    do {
        next = pud_addr_end(addr, end);
        if (remap_pmd_range(mm, pud, addr, next,
                pfn + (addr >> PAGE_SHIFT), prot))
            return -ENOMEM;
    } while (pud++, addr = next, addr != end);
    return 0;
}

function onAck(packet)
  rtt = now - packet.sendtime                      // 收包时间 减去 包中记录的发包时间就是RTT
  update_min_filter(RTpropFilter, rtt)             // 更新对 RTT 的估计
 
  delivered      += packet.size
  delivered_time =  now
  //计算当前实际的传输速率
  delivery_rate  =  (delivered - packet.delivered) / (delivered_time - packet.delivered_time)
 
  if (delivery_rate > BtlBwFilter.current_max      // 实际传输速率已经大于当前估计的瓶颈带宽，或
     || !packet.app_limited)                       // 不是应用受限（应用受限的样本对估计 BtlBw 无意义）
     update_max_filter(BtlBwFilter, delivery_rate) // 根更新对 BtlBw 的估计
 
  if (app_limited_until > 0)                       // 达到瓶颈带宽前，仍然可发送的字节数
     app_limited_until = app_limited_until - packet.size

function send(packet)
  bdp = BtlBwFilter.current_max * RTpropFilter.current_min  // 计算 BDP
  if (inflight >= cwnd_gain * bdp)                          // 如果正在传输中的数据量超过了允许的最大值
     return                                                 // 直接返回，接下来就等下一个 ACK，或者等超时重传

  // 能执行到这说明 inflight < cwnd_gain * bdp，即正在传输中的数据量 < 瓶颈容量
  if (now >= next_send_time)
     packet = nextPacketToSend()
     if (!packet)                      // 如果没有数据要发送
        app_limited_until = inflight   // 更新 “在达到瓶颈容量之前，仍然可发送的数据量”
        return

     packet.app_limited = (app_limited_until > 0)  // 如果仍然能发送若干字节才会达到瓶颈容量，说明处于 app_limited 状态
     packet.sendtime = now
     packet.delivered = delivered
     packet.delivered_time = delivered_time
     ship(packet)
     //下次发送数据的时间，通过这个控制拥塞
     next_send_time = now + packet.size / (pacing_gain * BtlBwFilter.current_max)
 //用定时器设置下次发送时间到期后的回调函数，就是继续执行send函数
  timerCallbackAt(send, next_send_time)

static struct tcp_congestion_ops tcp_bbr_cong_ops __read_mostly = {
    .flags        = TCP_CONG_NON_RESTRICTED,
    .name        = "bbr",
    .owner        = THIS_MODULE,
    .init        = bbr_init,
    .cong_control    = bbr_main,
    .sndbuf_expand    = bbr_sndbuf_expand,
    .undo_cwnd    = bbr_undo_cwnd,
    .cwnd_event    = bbr_cwnd_event,
    .ssthresh    = bbr_ssthresh,
    .tso_segs_goal    = bbr_tso_segs_goal,
    .get_info    = bbr_get_info,
    .set_state    = bbr_set_state,
};

static void bbr_main(struct sock *sk, const struct rate_sample *rs)
{
    struct bbr *bbr = inet_csk_ca(sk);
    u32 bw;
    
    bbr_update_model(sk, rs);

    bw = bbr_bw(sk);
    bbr_set_pacing_rate(sk, bw, bbr->pacing_gain);
    bbr_set_tso_segs_goal(sk);
    bbr_set_cwnd(sk, rs, rs->acked_sacked, bw, bbr->cwnd_gain);
}

/* Estimate the bandwidth based on how fast packets are delivered
   估算实际的带宽 
    1、更新RTT周期
    2、计算带宽=确认的字节数*BW_UNIT/采样时间
    3、带宽和minirtt样本加入新的rtt、bw样本
*/
static void bbr_update_bw(struct sock *sk, const struct rate_sample *rs)
{
    struct tcp_sock *tp = tcp_sk(sk);
    struct bbr *bbr = inet_csk_ca(sk);
    u64 bw;

    bbr->round_start = 0;
    if (rs->delivered < 0 || rs->interval_us <= 0)
        return; /* Not a valid observation */

    /* See if we've reached the next RTT */
    if (!before(rs->prior_delivered, bbr->next_rtt_delivered)) {
        bbr->next_rtt_delivered = tp->delivered;
        bbr->rtt_cnt++;
        bbr->round_start = 1;
        bbr->packet_conservation = 0;
    }
    bbr_lt_bw_sampling(sk, rs);

    /* Divide delivered by the interval to find a (lower bound) bottleneck
     * bandwidth sample. Delivered is in packets and interval_us in uS and
     * ratio will be <<1 for most connections. So delivered is first scaled.
     计算带宽
     */
    bw = (u64)rs->delivered * BW_UNIT;
    do_div(bw, rs->interval_us);

    /* If this sample is application-limited, it is likely to have a very
     * low delivered count that represents application behavior rather than
     * the available network rate. Such a sample could drag down estimated
     * bw, causing needless slow-down. Thus, to continue to send at the
     * last measured network rate, we filter out app-limited samples unless
     * they describe the path bw at least as well as our bw model.
     *
     * So the goal during app-limited phase is to proceed with the best
     * network rate no matter how long. We automatically leave this
     * phase when app writes faster than the network can deliver :)
     */
    if (!rs->is_app_limited || bw >= bbr_max_bw(sk)) {
        /* Incorporate new sample into our max bw filter. 
         带宽和minirtt样本加入新的rtt、bw样本*/
        minmax_running_max(&bbr->bw, bbr_bw_rtts, bbr->rtt_cnt, bw);
    }
}

/* Pace using current bw estimate and a gain factor. In order to help drive the
 * network toward lower queues while maintaining high utilization and low
 * latency, the average pacing rate aims to be slightly (~1%) lower than the
 * estimated bandwidth. This is an important aspect of the design. In this
 * implementation this slightly lower pacing rate is achieved implicitly by not
 * including link-layer headers in the packet size used for the pacing rate.
   
 */
static void bbr_set_pacing_rate(struct sock *sk, u32 bw, int gain)
{
    struct bbr *bbr = inet_csk_ca(sk);
    u64 rate = bw;

    rate = bbr_rate_bytes_per_sec(sk, rate, gain);
    rate = min_t(u64, rate, sk->sk_max_pacing_rate);
    if (bbr->mode != BBR_STARTUP || rate > sk->sk_pacing_rate)
        sk->sk_pacing_rate = rate;
}

/* The goal of PROBE_RTT mode is to have BBR flows cooperatively and
 * periodically drain the bottleneck queue, to converge to measure the true
 * min_rtt (unloaded propagation delay). This allows the flows to keep queues
 * small (reducing queuing delay and packet loss) and achieve fairness among
 * BBR flows.
 *
 * The min_rtt filter window is 10 seconds. When the min_rtt estimate expires,
 * we enter PROBE_RTT mode and cap the cwnd at bbr_cwnd_min_target=4 packets.
 * After at least bbr_probe_rtt_mode_ms=200ms and at least one packet-timed
 * round trip elapsed with that flight size <= 4, we leave PROBE_RTT mode and
 * re-enter the previous mode. BBR uses 200ms to approximately bound the
 * performance penalty of PROBE_RTT's cwnd capping to roughly 2% (200ms/10s).
 *
 * Note that flows need only pay 2% if they are busy sending over the last 10
 * seconds. Interactive applications (e.g., Web, RPCs, video chunks) often have
 * natural silences or low-rate periods within 10 seconds where the rate is low
 * enough for long enough to drain its queue in the bottleneck. We pick up
 * these min RTT measurements opportunistically with our min_rtt filter. :-)
 */
static void bbr_update_min_rtt(struct sock *sk, const struct rate_sample *rs)
{
    struct tcp_sock *tp = tcp_sk(sk);
    struct bbr *bbr = inet_csk_ca(sk);
    bool filter_expired;

    /* Track min RTT seen in the min_rtt_win_sec filter window: */
    filter_expired = after(tcp_time_stamp,
                   bbr->min_rtt_stamp + bbr_min_rtt_win_sec * HZ);
    if (rs->rtt_us >= 0 &&
        (rs->rtt_us <= bbr->min_rtt_us || filter_expired)) {
        bbr->min_rtt_us = rs->rtt_us;
        bbr->min_rtt_stamp = tcp_time_stamp;
    }

    if (bbr_probe_rtt_mode_ms > 0 && filter_expired &&
        !bbr->idle_restart && bbr->mode != BBR_PROBE_RTT) {
        bbr->mode = BBR_PROBE_RTT;  /* dip, drain queue */
        bbr->pacing_gain = BBR_UNIT;
        bbr->cwnd_gain = BBR_UNIT;
        bbr_save_cwnd(sk);  /* note cwnd so we can restore it */
        bbr->probe_rtt_done_stamp = 0;
    }

    if (bbr->mode == BBR_PROBE_RTT) {//如果是probe_rtt状态
        /* Ignore low rate samples during this mode. */
        tp->app_limited =
            (tp->delivered + tcp_packets_in_flight(tp)) ? : 1;
        /* Maintain min packets in flight for max(200 ms, 1 round). */
        if (!bbr->probe_rtt_done_stamp &&
            tcp_packets_in_flight(tp) <= bbr_cwnd_min_target) {
            bbr->probe_rtt_done_stamp = tcp_time_stamp +
                msecs_to_jiffies(bbr_probe_rtt_mode_ms);
            bbr->probe_rtt_round_done = 0;
            bbr->next_rtt_delivered = tp->delivered;
        } else if (bbr->probe_rtt_done_stamp) {
            if (bbr->round_start)
                bbr->probe_rtt_round_done = 1;
            if (bbr->probe_rtt_round_done &&
                after(tcp_time_stamp, bbr->probe_rtt_done_stamp)) {
                bbr->min_rtt_stamp = tcp_time_stamp;
                bbr->restore_cwnd = 1;  /* snap to prior_cwnd */
                bbr_reset_mode(sk);
            }
        }
    }
    bbr->idle_restart = 0;
}

/* This will initiate an outgoing connection. */
int tcp_v4_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len)
{
    struct sockaddr_in *usin = (struct sockaddr_in *)uaddr;
    struct inet_sock *inet = inet_sk(sk);
    struct tcp_sock *tp = tcp_sk(sk);
    __be16 orig_sport, orig_dport;
    __be32 daddr, nexthop;
    struct flowi4 *fl4;
    struct rtable *rt;
    int err;
    struct ip_options_rcu *inet_opt;

    if (addr_len < sizeof(struct sockaddr_in))
        return -EINVAL;
    /*AF_INET是用户调用socket函数创建套接字时传入的参数，
      这里校验地址长度和协议簇*/
    if (usin->sin_family != AF_INET)
        return -EAFNOSUPPORT;
    /*将下一跳地址和目的地址的临时变量都暂时设为用户提交的地址*/
    nexthop = daddr = usin->sin_addr.s_addr;
    inet_opt = rcu_dereference_protected(inet->inet_opt,
                         lockdep_sock_is_held(sk));
    /*  如果使用了来源地址路由，选择一个合适的下一跳地址。*/
    if (inet_opt && inet_opt->opt.srr) {
        if (!daddr)
            return -EINVAL;
        nexthop = inet_opt->opt.faddr;
    }
    /*获取数据包的ip层路由信息*/
    orig_sport = inet->inet_sport;
    orig_dport = usin->sin_port;
    fl4 = &inet->cork.fl.u.ip4;
    /*一台主机可能有多个ip地址，用哪个ip地址发送数据包了？
      根据nexthop参数(也就是connect传递下来的服务器ip)查路由表，
      命中的路由表项中包含有本地ip地址*/
    rt = ip_route_connect(fl4, nexthop, inet->inet_saddr,
                  RT_CONN_FLAGS(sk), sk->sk_bound_dev_if,
                  IPPROTO_TCP,
                  orig_sport, orig_dport, sk);
    if (IS_ERR(rt)) {
        err = PTR_ERR(rt);
        if (err == -ENETUNREACH)
            IP_INC_STATS(sock_net(sk), IPSTATS_MIB_OUTNOROUTES);
        return err;
    }
    /* 进行路由查找，并校验返回的路由的类型，
        TCP是不被允许使用多播和广播的。*/
    if (rt->rt_flags & (RTCF_MULTICAST | RTCF_BROADCAST)) {
        ip_rt_put(rt);
        return -ENETUNREACH;
    }
    
    if (!inet_opt || !inet_opt->opt.srr)
        daddr = fl4->daddr;
    /*如果用户之前没有bind源，inet->inet_saddr将会是0.
      此处判断如果saddr是0，就把查路由返回的fl4->saddr赋值给inet->inet_saddr。
      inet->inet_saddr的值将来会作为syn报文的源ip；
      
      1、如果在这里更改源ip，用随机数填充后发送给服务器，
      服务器会不会不停地发送ack数据包导致达到DDOS的效果了?
      2、如果把这里的源ip改成受害服务器的地址，然后随机发给第三方服务器，第三方服务器收到
      SYN数据包后分分给受害服务器发送ack数据包，是不是达到了反射DDOS的效果了？
    */
    if (!inet->inet_saddr)
        inet->inet_saddr = fl4->saddr;
    sk_rcv_saddr_set(sk, inet->inet_saddr);

    if (tp->rx_opt.ts_recent_stamp && inet->inet_daddr != daddr) {
        /* Reset inherited state */
        tp->rx_opt.ts_recent       = 0;
        tp->rx_opt.ts_recent_stamp = 0;
        if (likely(!tp->repair))
            tp->write_seq       = 0;
    }

    if (tcp_death_row.sysctl_tw_recycle &&
        !tp->rx_opt.ts_recent_stamp && fl4->daddr == daddr)
        tcp_fetch_timewait_stamp(sk, &rt->dst);
    
    inet->inet_dport = usin->sin_port;
    sk_daddr_set(sk, daddr);

    inet_csk(sk)->icsk_ext_hdr_len = 0;
    if (inet_opt)
        inet_csk(sk)->icsk_ext_hdr_len = inet_opt->opt.optlen;

    tp->rx_opt.mss_clamp = TCP_MSS_DEFAULT;

    /* Socket identity is still unknown (sport may be zero).
     * However we set state to SYN-SENT and not releasing socket
     * lock select source port, enter ourselves into the hash tables and
     * complete initialization after this.
       将TCP的状态设为TCP_SYN_SENT，动态绑定一个本地端口
     */
    tcp_set_state(sk, TCP_SYN_SENT);
    err = inet_hash_connect(&tcp_death_row, sk);
    if (err)
        goto failure;

    sk_set_txhash(sk);
    /*最终调用的是fib_table_lookup函数从trie树中查找最长匹配的ip地址*/
    rt = ip_route_newports(fl4, rt, orig_sport, orig_dport,
                   inet->inet_sport, inet->inet_dport, sk);
    if (IS_ERR(rt)) {
        err = PTR_ERR(rt);
        rt = NULL;
        goto failure;
    }
    /* OK, now commit destination to socket.  */
    sk->sk_gso_type = SKB_GSO_TCPV4;
    sk_setup_caps(sk, &rt->dst);

    if (!tp->write_seq && likely(!tp->repair))
        tp->write_seq = secure_tcp_sequence_number(inet->inet_saddr,
                               inet->inet_daddr,
                               inet->inet_sport,
                               usin->sin_port);

    inet->inet_id = tp->write_seq ^ jiffies;
    /*生成SYN数据包并发送*/
    err = tcp_connect(sk);

    rt = NULL;
    if (err)
        goto failure;

    return 0;

failure:
    /*
     * This unhashes the socket and releases the local port,
     * if necessary.
     */
    tcp_set_state(sk, TCP_CLOSE);
    ip_rt_put(rt);
    sk->sk_route_caps = 0;
    inet->inet_dport = 0;
    return err;
}

/* This routine actually transmits TCP packets queued in by
 * tcp_do_sendmsg().  This is used by both the initial
 * transmission and possible later retransmissions.
 * All SKB's seen here are completely headerless.  It is our
 * job to build the TCP header, and pass the packet down to
 * IP so it can do the same plus pass the packet off to the
 * device.
 *
 * We are working here with either a clone of the original
 * SKB, or a fresh unique copy made by the retransmit engine.
 复制或者拷贝skb，构造skb中的tcp首部，并将调用网络层的发送函数发送skb；
 在发送前，首先需要克隆或者复制skb，因为在成功发送到网络设备之后，skb会释放，
 而tcp层不能真正的释放，是需要等到对该数据段的ack才可以释放；然后构造tcp首部和选项；
 最后调用网络层提供的发送回调函数发送skb，ip层的回调函数为ip_queue_xmit
 */
static int tcp_transmit_skb(struct sock *sk, struct sk_buff *skb, int clone_it,
                gfp_t gfp_mask)
{
    const struct inet_connection_sock *icsk = inet_csk(sk);
    struct inet_sock *inet;
    struct tcp_sock *tp;
    struct tcp_skb_cb *tcb;
    struct tcp_out_options opts;
    unsigned int tcp_options_size, tcp_header_size;
    struct tcp_md5sig_key *md5;
    struct tcphdr *th;
    int err;

    BUG_ON(!skb || !tcp_skb_pcount(skb));
    tp = tcp_sk(sk);
    //如果还有其他进程使用skb，就需要复制skb
    if (clone_it) {
        skb_mstamp_get(&skb->skb_mstamp);
        TCP_SKB_CB(skb)->tx.in_flight = TCP_SKB_CB(skb)->end_seq
            - tp->snd_una;
        tcp_rate_skb_sent(sk, skb);

        if (unlikely(skb_cloned(skb)))
            skb = pskb_copy(skb, gfp_mask);
        else
            skb = skb_clone(skb, gfp_mask);
        if (unlikely(!skb))
            return -ENOBUFS;
    }

    inet = inet_sk(sk);
    tcb = TCP_SKB_CB(skb);
    memset(&opts, 0, sizeof(opts));
    //是否是SYN请求数据包
    if (unlikely(tcb->tcp_flags & TCPHDR_SYN))
        //构建TCP选项包括时间戳、窗口大小、选择回答SACK
        tcp_options_size = tcp_syn_options(sk, skb, &opts, &md5);
    else//构建常规TCP选项
        tcp_options_size = tcp_established_options(sk, skb, &opts,
                               &md5);
    //tCP头部长度包括选择长度+ TCP头部
    tcp_header_size = tcp_options_size + sizeof(struct tcphdr);

    /* if no packet is in qdisc/device queue, then allow XPS to select
     * another queue. We can be called from tcp_tsq_handler()
     * which holds one reference to sk_wmem_alloc.
     *
     * TODO: Ideally, in-flight pure ACK packets should not matter here.
     * One way to get this would be to set skb->truesize = 2 on them.
     */
    skb->ooo_okay = sk_wmem_alloc_get(sk) < SKB_TRUESIZE(1);

    skb_push(skb, tcp_header_size);
    skb_reset_transport_header(skb);

    skb_orphan(skb);
    skb->sk = sk;
    skb->destructor = skb_is_tcp_pure_ack(skb) ? __sock_wfree : tcp_wfree;
    skb_set_hash_from_sk(skb, sk);
    atomic_add(skb->truesize, &sk->sk_wmem_alloc);

    /* Build TCP header and checksum it. 
    前面做了大量的准备工作，这里终于开始构造tcp包头了
    */
    th = (struct tcphdr *)skb->data;
    th->source        = inet->inet_sport;
    th->dest        = inet->inet_dport;
    th->seq            = htonl(tcb->seq);
    th->ack_seq        = htonl(tp->rcv_nxt);
    *(((__be16 *)th) + 6)    = htons(((tcp_header_size >> 2) << 12) |
                    tcb->tcp_flags);

    th->check        = 0;
    th->urg_ptr        = 0;

    /* The urg_mode check is necessary during a below snd_una win probe */
    //SYN包不需要计算窗口
    if (unlikely(tcp_urg_mode(tp) && before(tcb->seq, tp->snd_up))) {
        if (before(tp->snd_up, tcb->seq + 0x10000)) {
            th->urg_ptr = htons(tp->snd_up - tcb->seq);
            th->urg = 1;
        } else if (after(tcb->seq + 0xFFFF, tp->snd_nxt)) {
            th->urg_ptr = htons(0xFFFF);
            th->urg = 1;
        }
    }

    tcp_options_write((__be32 *)(th + 1), tp, &opts);
    skb_shinfo(skb)->gso_type = sk->sk_gso_type;
    if (likely(!(tcb->tcp_flags & TCPHDR_SYN))) {
        th->window      = htons(tcp_select_window(sk));
        tcp_ecn_send(sk, skb, th, tcp_header_size);
    } else {
        /* RFC1323: The window in SYN & SYN/ACK segments
         * is never scaled.
         */
        th->window    = htons(min(tp->rcv_wnd, 65535U));
    }
#ifdef CONFIG_TCP_MD5SIG
    /* Calculate the MD5 hash, as we have all we need now */
    if (md5) {
        sk_nocaps_add(sk, NETIF_F_GSO_MASK);
        tp->af_specific->calc_md5_hash(opts.hash_location,
                           md5, sk, skb);
    }
#endif

    icsk->icsk_af_ops->send_check(sk, skb);

    if (likely(tcb->tcp_flags & TCPHDR_ACK))
        tcp_event_ack_sent(sk, tcp_skb_pcount(skb));//清楚定时器
    /* 有数据要发送 */
    if (skb->len != tcp_header_size) {
        tcp_event_data_sent(tp, sk);
        tp->data_segs_out += tcp_skb_pcount(skb);
    }
    /* 统计分段数 */
    if (after(tcb->end_seq, tp->snd_nxt) || tcb->seq == tcb->end_seq)
        TCP_ADD_STATS(sock_net(sk), TCP_MIB_OUTSEGS,
                  tcp_skb_pcount(skb));

    tp->segs_out += tcp_skb_pcount(skb);
    /* OK, its time to fill skb_shinfo(skb)->gso_{segs|size} */
    /* skb中分段数统计 */
    skb_shinfo(skb)->gso_segs = tcp_skb_pcount(skb);
    skb_shinfo(skb)->gso_size = tcp_skb_mss(skb);

    /* Our usage of tstamp should remain private */
    skb->tstamp.tv64 = 0;

    /* Cleanup our debris for IP stacks */
    /* 清空tcb，ip层要使用 */
    memset(skb->cb, 0, max(sizeof(struct inet_skb_parm),
                   sizeof(struct inet6_skb_parm)));
    //数据包给ip层继续添加ip地址；函数指针实际指向ip_queue_ximit，这也是实际调用的ip层函数
    err = icsk->icsk_af_ops->queue_xmit(sk, skb, &inet->cork.fl);

    if (likely(err <= 0))
        return err;
    /* 拥塞控制 */
    tcp_enter_cwr(sk);

    return net_xmit_eval(err);
}

/* Note: skb->sk can be different from sk, in case of tunnels */
int ip_queue_xmit(struct sock *sk, struct sk_buff *skb, struct flowi *fl)
{
    struct inet_sock *inet = inet_sk(sk);
    struct net *net = sock_net(sk);
    struct ip_options_rcu *inet_opt;
    struct flowi4 *fl4;
    struct rtable *rt;
    struct iphdr *iph;
    int res;

    /* Skip all of this if the packet is already routed,
     * f.e. by something like SCTP.
     */
    rcu_read_lock();
    inet_opt = rcu_dereference(inet->inet_opt);
    fl4 = &fl->u.ip4;
    /* 获取skb中的路由缓存 */
    rt = skb_rtable(skb);
    if (rt)
        goto packet_routed;

    /* Make sure we can route this packet. */
    /* 检查控制块中的路由缓存 */
    rt = (struct rtable *)__sk_dst_check(sk, 0);
    /* 缓存过期 */
    if (!rt) {
        __be32 daddr;

        /* Use correct destination address if we have options. */
        //终于看到了目的ip地址
        daddr = inet->inet_daddr;
        if (inet_opt && inet_opt->opt.srr)
            daddr = inet_opt->opt.faddr;

        /* If this fails, retransmit mechanism of transport layer will
         * keep trying until route appears or the connection times
         * itself out. 重新查找路由缓存
         */
        rt = ip_route_output_ports(net, fl4, sk,
                       daddr, inet->inet_saddr,
                       inet->inet_dport,
                       inet->inet_sport,
                       sk->sk_protocol,
                       RT_CONN_FLAGS(sk),
                       sk->sk_bound_dev_if);
        if (IS_ERR(rt))
            goto no_route;
        /* 设置控制块的路由缓存 */
        sk_setup_caps(sk, &rt->dst);
    }
     /* 将路由设置到skb中 */
    skb_dst_set_noref(skb, &rt->dst);

packet_routed:
    if (inet_opt && inet_opt->opt.is_strictroute && rt->rt_uses_gateway)
        goto no_route;

    /* OK, we know where to send it, allocate and build IP header. */
    /*找到目标后开始在tcp包的基础上构造ip包*/
    /*在skb中加上ip的头*/
    skb_push(skb, sizeof(struct iphdr) + (inet_opt ? inet_opt->opt.optlen : 0));
    skb_reset_network_header(skb);
    iph = ip_hdr(skb);
    *((__be16 *)iph) = htons((4 << 12) | (5 << 8) | (inet->tos & 0xff));
    if (ip_dont_fragment(sk, &rt->dst) && !skb->ignore_df)
        iph->frag_off = htons(IP_DF);
    else
        iph->frag_off = 0;
    iph->ttl      = ip_select_ttl(inet, &rt->dst);
    iph->protocol = sk->sk_protocol;
    ip_copy_addrs(iph, fl4);

    /* Transport layer set skb->h.foo itself. */
    /* 构造ip选项 */
    if (inet_opt && inet_opt->opt.optlen) {
        iph->ihl += inet_opt->opt.optlen >> 2;
        ip_options_build(skb, &inet_opt->opt, inet->inet_daddr, rt, 0);
    }
    /* 设置id */
    ip_select_ident_segs(net, skb, sk,
                 skb_shinfo(skb)->gso_segs ?: 1);

    /* TODO : should we use skb->sk here instead of sk ? */
    skb->priority = sk->sk_priority;
    skb->mark = sk->sk_mark;
    //发送ip包
    res = ip_local_out(net, sk, skb);
    rcu_read_unlock();
    return res;

no_route://无路由处理
    rcu_read_unlock();
    IP_INC_STATS(net, IPSTATS_MIB_OUTNOROUTES);
    kfree_skb(skb);
    return -EHOSTUNREACH;
}

/* Output packet to network from transport.  */
static inline int dst_output(struct net *net, struct sock *sk, struct sk_buff *skb)
{
    return skb_dst(skb)->output(net, sk, skb);
}

/*
 *    Send a SYN-ACK after having received a SYN.
 *    This still operates on a request_sock only, not on a big
 *    socket.
 */
static int tcp_v4_send_synack(const struct sock *sk, struct dst_entry *dst,
                  struct flowi *fl,
                  struct request_sock *req,
                  struct tcp_fastopen_cookie *foc,
                  enum tcp_synack_type synack_type)
{
    const struct inet_request_sock *ireq = inet_rsk(req);
    struct flowi4 fl4;
    int err = -1;
    struct sk_buff *skb;

    /* First, grab a route. */
    if (!dst && (dst = inet_csk_route_req(sk, &fl4, req)) == NULL)
        return -1;
    /*构造syn-sck的包，并返回skb*/
    skb = tcp_make_synack(sk, dst, req, foc, synack_type);

    if (skb) {
        __tcp_v4_send_check(skb, ireq->ir_loc_addr, ireq->ir_rmt_addr);
        /*添加ip包头并发送*/
        err = ip_build_and_send_pkt(skb, sk, ireq->ir_loc_addr,
                        ireq->ir_rmt_addr,
                        ireq->opt);
        err = net_xmit_eval(err);
    }

    return err;
}

/**
 * tcp_make_synack - Prepare a SYN-ACK.
 * sk: listener socket
 * dst: dst entry attached to the SYNACK
 * req: request_sock pointer
 *
 * Allocate one skb and build a SYNACK packet.
 * @dst is consumed : Caller should not use it again.
   生成SYN-ACK数据包
 */
struct sk_buff *tcp_make_synack(const struct sock *sk, struct dst_entry *dst,
                struct request_sock *req,
                struct tcp_fastopen_cookie *foc,
                enum tcp_synack_type synack_type)
{
    struct inet_request_sock *ireq = inet_rsk(req);
    const struct tcp_sock *tp = tcp_sk(sk);
    struct tcp_md5sig_key *md5 = NULL;
    struct tcp_out_options opts;
    struct sk_buff *skb;
    int tcp_header_size;
    struct tcphdr *th;
    u16 user_mss;
    int mss;
    /*分配skb，这是DDOS攻击生效的原因之一：
    如果一个client不停的更改源ip给server发送syn包，server还以为有很多
    client想和自己通信，然后不停地分配skb为
    接下来的通信做准备，可能导致内存耗尽
    */
    skb = alloc_skb(MAX_TCP_HEADER, GFP_ATOMIC);
    if (unlikely(!skb)) {
        dst_release(dst);
        return NULL;
    }
    /* Reserve space for headers. */
    skb_reserve(skb, MAX_TCP_HEADER);

    switch (synack_type) {
    case TCP_SYNACK_NORMAL:
        skb_set_owner_w(skb, req_to_sk(req));
        break;
    case TCP_SYNACK_COOKIE:
        /* Under synflood, we do not attach skb to a socket,
         * to avoid false sharing.
         */
        break;
    case TCP_SYNACK_FASTOPEN:
        /* sk is a const pointer, because we want to express multiple
         * cpu might call us concurrently.
         * sk->sk_wmem_alloc in an atomic, we can promote to rw.
         */
        skb_set_owner_w(skb, (struct sock *)sk);
        break;
    }
    skb_dst_set(skb, dst);

    mss = dst_metric_advmss(dst);
    user_mss = READ_ONCE(tp->rx_opt.user_mss);
    if (user_mss && user_mss < mss)
        mss = user_mss;

    memset(&opts, 0, sizeof(opts));
#ifdef CONFIG_SYN_COOKIES
    if (unlikely(req->cookie_ts))
        skb->skb_mstamp.stamp_jiffies = cookie_init_timestamp(req);
    else
#endif
    skb_mstamp_get(&skb->skb_mstamp);

#ifdef CONFIG_TCP_MD5SIG
    rcu_read_lock();
    md5 = tcp_rsk(req)->af_specific->req_md5_lookup(sk, req_to_sk(req));
#endif
    skb_set_hash(skb, tcp_rsk(req)->txhash, PKT_HASH_TYPE_L4);
    tcp_header_size = tcp_synack_options(req, mss, skb, &opts, md5, foc) +
              sizeof(*th);
    //开始填充tcp头部了
    skb_push(skb, tcp_header_size);
    skb_reset_transport_header(skb);

    th = (struct tcphdr *)skb->data;
    memset(th, 0, sizeof(struct tcphdr));
    th->syn = 1;
    th->ack = 1;
    tcp_ecn_make_synack(req, th);
    th->source = htons(ireq->ir_num);
    th->dest = ireq->ir_rmt_port;
    /* Setting of flags are superfluous here for callers (and ECE is
     * not even correctly set)
     */
    tcp_init_nondata_skb(skb, tcp_rsk(req)->snt_isn,
                 TCPHDR_SYN | TCPHDR_ACK);

    th->seq = htonl(TCP_SKB_CB(skb)->seq);
    /* XXX data is queued and acked as is. No buffer/window check */
    th->ack_seq = htonl(tcp_rsk(req)->rcv_nxt);

    /* RFC1323: The window in SYN & SYN/ACK segments is never scaled. */
    th->window = htons(min(req->rsk_rcv_wnd, 65535U));
    tcp_options_write((__be32 *)(th + 1), NULL, &opts);
    th->doff = (tcp_header_size >> 2);
    __TCP_INC_STATS(sock_net(sk), TCP_MIB_OUTSEGS);

#ifdef CONFIG_TCP_MD5SIG
    /* Okay, we have all we need - do the md5 hash if needed */
    if (md5)
        tcp_rsk(req)->af_specific->calc_md5_hash(opts.hash_location,
                           md5, req_to_sk(req), skb);
    rcu_read_unlock();
#endif

    /* Do not fool tcpdump (if any), clean our debris */
    skb->tstamp.tv64 = 0;
    return skb;
}
EXPORT_SYMBOL(tcp_make_synack);

/*
 *        Add an ip header to a skbuff and send it out.
 *
 */
int ip_build_and_send_pkt(struct sk_buff *skb, const struct sock *sk,
              __be32 saddr, __be32 daddr, struct ip_options_rcu *opt)
{
    struct inet_sock *inet = inet_sk(sk);
    struct rtable *rt = skb_rtable(skb);
    struct net *net = sock_net(sk);
    struct iphdr *iph;

    /* Build the IP header. 构造ip头*/
    skb_push(skb, sizeof(struct iphdr) + (opt ? opt->opt.optlen : 0));
    skb_reset_network_header(skb);
    iph = ip_hdr(skb);
    iph->version  = 4;
    iph->ihl      = 5;
    iph->tos      = inet->tos;
    iph->ttl      = ip_select_ttl(inet, &rt->dst);
    iph->daddr    = (opt && opt->opt.srr ? opt->opt.faddr : daddr);
    iph->saddr    = saddr;
    iph->protocol = sk->sk_protocol;
    if (ip_dont_fragment(sk, &rt->dst)) {
        iph->frag_off = htons(IP_DF);
        iph->id = 0;
    } else {
        iph->frag_off = 0;
        __ip_select_ident(net, iph, 1);
    }

    if (opt && opt->opt.optlen) {
        iph->ihl += opt->opt.optlen>>2;
        ip_options_build(skb, &opt->opt, daddr, rt, 0);
    }

    skb->priority = sk->sk_priority;
    skb->mark = sk->sk_mark;

    /* Send it out. */
    return ip_local_out(net, skb->sk, skb);
}

/* This routine sends an ack and also updates the window. */
void tcp_send_ack(struct sock *sk)
{
    struct sk_buff *buff;

    /* If we have been reset, we may not send again. */
    /* 如果当前的套接字已经被关闭了，那么直接返回。 */
    if (sk->sk_state == TCP_CLOSE)
        return;
    /* 拥塞避免 */
    tcp_ca_event(sk, CA_EVENT_NON_DELAYED_ACK);

    /* We are not putting this on the write queue, so
     * tcp_transmit_skb() will set the ownership to this
     * sock.
     */
    buff = alloc_skb(MAX_TCP_HEADER,
             sk_gfp_mask(sk, GFP_ATOMIC | __GFP_NOWARN));
    if (unlikely(!buff)) {
        inet_csk_schedule_ack(sk);
        inet_csk(sk)->icsk_ack.ato = TCP_ATO_MIN;
        inet_csk_reset_xmit_timer(sk, ICSK_TIME_DACK,
                      TCP_DELACK_MAX, TCP_RTO_MAX);
        return;
    }

    /* Reserve space for headers and prepare control bits. */
    /* 初始化 ACK 包 */
    skb_reserve(buff, MAX_TCP_HEADER);
    tcp_init_nondata_skb(buff, tcp_acceptable_seq(sk), TCPHDR_ACK);

    /* We do not want pure acks influencing TCP Small Queues or fq/pacing
     * too much.
     * SKB_TRUESIZE(max(1 .. 66, MAX_TCP_HEADER)) is unfortunately ~784
     * We also avoid tcp_wfree() overhead (cache line miss accessing
     * tp->tsq_flags) by using regular sock_wfree()
     */
    skb_set_tcp_pure_ack(buff);

    /* Send it off, this clears delayed acks for us. */
    /* 添加时间戳并发送 ACK 包 */
    skb_mstamp_get(&buff->skb_mstamp);
    //还是从这里构造和发送ack包，老演员了！
    tcp_transmit_skb(sk, buff, 0, (__force gfp_t)0);
}
EXPORT_SYMBOL_GPL(tcp_send_ack);

/** 
 *    struct sk_buff - socket buffer
 *    @next: Next buffer in list
 *    @prev: Previous buffer in list
 *    @tstamp: Time we arrived/left
 *    @rbnode: RB tree node, alternative to next/prev for netem/tcp
 *    @sk: Socket we are owned by
 *    @dev: Device we arrived on/are leaving by
 *    @cb: Control buffer. Free for use by every layer. Put private vars here
 *    @_skb_refdst: destination entry (with norefcount bit)
 *    @sp: the security path, used for xfrm
 *    @len: Length of actual data
 *    @data_len: Data length
 *    @mac_len: Length of link layer header
 *    @hdr_len: writable header length of cloned skb
 *    @csum: Checksum (must include start/offset pair)
 *    @csum_start: Offset from skb->head where checksumming should start
 *    @csum_offset: Offset from csum_start where checksum should be stored
 *    @priority: Packet queueing priority
 *    @ignore_df: allow local fragmentation
 *    @cloned: Head may be cloned (check refcnt to be sure)
 *    @ip_summed: Driver fed us an IP checksum
 *    @nohdr: Payload reference only, must not modify header
 *    @nfctinfo: Relationship of this skb to the connection
 *    @pkt_type: Packet class
 *    @fclone: skbuff clone status
 *    @ipvs_property: skbuff is owned by ipvs
 *    @peeked: this packet has been seen already, so stats have been
 *        done for it, don't do them again
 *    @nf_trace: netfilter packet trace flag
 *    @protocol: Packet protocol from driver
 *    @destructor: Destruct function
 *    @nfct: Associated connection, if any
 *    @nf_bridge: Saved data about a bridged frame - see br_netfilter.c
 *    @skb_iif: ifindex of device we arrived on
 *    @tc_index: Traffic control index
 *    @tc_verd: traffic control verdict
 *    @hash: the packet hash
 *    @queue_mapping: Queue mapping for multiqueue devices
 *    @xmit_more: More SKBs are pending for this queue
 *    @ndisc_nodetype: router type (from link layer)
 *    @ooo_okay: allow the mapping of a socket to a queue to be changed
 *    @l4_hash: indicate hash is a canonical 4-tuple hash over transport
 *        ports.
 *    @sw_hash: indicates hash was computed in software stack
 *    @wifi_acked_valid: wifi_acked was set
 *    @wifi_acked: whether frame was acked on wifi or not
 *    @no_fcs:  Request NIC to treat last 4 bytes as Ethernet FCS
  *    @napi_id: id of the NAPI struct this skb came from
 *    @secmark: security marking
 *    @mark: Generic packet mark
 *    @vlan_proto: vlan encapsulation protocol
 *    @vlan_tci: vlan tag control information
 *    @inner_protocol: Protocol (encapsulation)
 *    @inner_transport_header: Inner transport layer header (encapsulation)
 *    @inner_network_header: Network layer header (encapsulation)
 *    @inner_mac_header: Link layer header (encapsulation)
 *    @transport_header: Transport layer header
 *    @network_header: Network layer header
 *    @mac_header: Link layer header
 *    @tail: Tail pointer
 *    @end: End pointer
 *    @head: Head of buffer
 *    @data: Data head pointer
 *    @truesize: Buffer size
 *    @users: User count - see {datagram,tcp}.c
 */

struct sk_buff {
    union {
        struct {
            /* These two members must be first. */
            /*双向链表结构，用来存储网络数据包*/
            struct sk_buff        *next;
            struct sk_buff        *prev;

            union {
                /*报文到达或者离开的时间戳； Time we arrived 表示这个skb的接收到的时间，
                一般是在包从驱动中往二层发送的接口函数中设置 */
                ktime_t        tstamp;
                struct skb_mstamp skb_mstamp;
            };
        };
        /**/
        struct rb_node    rbnode; /* used in netem & tcp stack */
    };
    struct sock        *sk;//该数据包属于哪个socket
    struct net_device    *dev;//收到这个报文的设备

    /*
     * This is the control buffer. It is free to use for every
     * layer. Please put your private variables there. If you
     * want to keep them across layers you have to do a skb_clone()
     * first. This is owned by whoever has the skb queued ATM.
     */
    char            cb[48] __aligned(8);

    unsigned long        _skb_refdst;
    //析构函数，一般都是设置为sock_rfree或者sock_wfree
    void            (*destructor)(struct sk_buff *skb);
#ifdef CONFIG_XFRM
    struct    sec_path    *sp;
#endif
#if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)
    struct nf_conntrack    *nfct;
#endif
#if IS_ENABLED(CONFIG_BRIDGE_NETFILTER)
    struct nf_bridge_info    *nf_bridge;
#endif
    /*表示当前的skb中的数据的长度，这个长度即包括buf中的数据也包括切片的数据，
    也就是保存在skb_shared_info中的数据*/
    unsigned int        len,
                data_len;//只表示切片数据的长度，也就是skb_shared_info中的长度
    __u16            mac_len,//mac头的长度
                hdr_len;//用于clone的时候，它表示clone的skb的头的长度

    /* Following fields are _not_ copied in __copy_skb_header()
     * Note that queue_mapping is here mostly to fill a hole.
     */
    kmemcheck_bitfield_begin(flags1);
    __u16            queue_mapping;//多队列设备的映射，也就是说映射到那个队列。 

/* if you move cloned around you also must adapt those constants */
#ifdef __BIG_ENDIAN_BITFIELD
#define CLONED_MASK    (1 << 7)
#else
#define CLONED_MASK    1
#endif
#define CLONED_OFFSET()        offsetof(struct sk_buff, __cloned_offset)

    __u8            __cloned_offset[0];
    __u8            cloned:1,
                nohdr:1,
                fclone:2,
                peeked:1,
                head_frag:1,
                xmit_more:1,
                __unused:1; /* one bit hole */
    kmemcheck_bitfield_end(flags1);

    /* fields enclosed in headers_start/headers_end are copied
     * using a single memcpy() in __copy_skb_header()
     */
    /* private: */
    __u32            headers_start[0];
    /* public: */

/* if you move pkt_type around you also must adapt those constants */
#ifdef __BIG_ENDIAN_BITFIELD
#define PKT_TYPE_MAX    (7 << 5)
#else
#define PKT_TYPE_MAX    7
#endif
#define PKT_TYPE_OFFSET()    offsetof(struct sk_buff, __pkt_type_offset)

    __u8            __pkt_type_offset[0];
    __u8            pkt_type:3;
    __u8            pfmemalloc:1;
    __u8            ignore_df:1;
    __u8            nfctinfo:3;

    __u8            nf_trace:1;
    __u8            ip_summed:2;
    __u8            ooo_okay:1;
    __u8            l4_hash:1;
    __u8            sw_hash:1;
    __u8            wifi_acked_valid:1;
    __u8            wifi_acked:1;

    __u8            no_fcs:1;
    /* Indicates the inner headers are valid in the skbuff. */
    __u8            encapsulation:1;
    __u8            encap_hdr_csum:1;
    __u8            csum_valid:1;
    __u8            csum_complete_sw:1;
    __u8            csum_level:2;
    __u8            csum_bad:1;

#ifdef CONFIG_IPV6_NDISC_NODETYPE
    __u8            ndisc_nodetype:2;
#endif
    __u8            ipvs_property:1;
    __u8            inner_protocol_type:1;
    __u8            remcsum_offload:1;
#ifdef CONFIG_NET_SWITCHDEV
    __u8            offload_fwd_mark:1;
#endif
    /* 2, 4 or 5 bit hole */

#ifdef CONFIG_NET_SCHED
    __u16            tc_index;    /* traffic control index */
#ifdef CONFIG_NET_CLS_ACT
    __u16            tc_verd;    /* traffic control verdict */
#endif
#endif

    union {
        __wsum        csum;
        struct {
            __u16    csum_start;
            __u16    csum_offset;
        };
    };
    __u32            priority;/*优先级，主要用于QOS*/
    int            skb_iif;
    __u32            hash;
    __be16            vlan_proto;
    __u16            vlan_tci;
#if defined(CONFIG_NET_RX_BUSY_POLL) || defined(CONFIG_XPS)
    union {
        unsigned int    napi_id;
        unsigned int    sender_cpu;
    };
#endif
#ifdef CONFIG_NETWORK_SECMARK
    __u32        secmark;
#endif

    union {
        __u32        mark;
        __u32        reserved_tailroom;
    };

    union {
        __be16        inner_protocol;
        __u8        inner_ipproto;
    };

    __u16            inner_transport_header;
    __u16            inner_network_header;
    __u16            inner_mac_header;

    __be16            protocol;//协议类型
    __u16            transport_header;//传输层头部
    __u16            network_header;//网络层头部
    __u16            mac_header;//链路层头部

    /* private: */
    __u32            headers_end[0];
    /* public: */

    /* These elements must be at the end, see alloc_skb() for details. 
    sk_buff_data_t就是unsigned char *
    */
    sk_buff_data_t        tail;//指向报文尾巴
    sk_buff_data_t        end;//指向报文最后一个字节
    unsigned char        *head,//分配的内存块的起始位置；指向数据区中开始的位置（非实际数据区域开始位置）
                *data;//保存数据内容的首地址；（实际数据区域开始位置）
    /*缓冲区的总长度，包括sk_buff结构和数据部分。
    如果申请一个len字节的缓冲区，alloc_skb函数会把它初始化成len+sizeof(sk_buff)。
    当skb->len变化时，这个变量也会变化*/
    unsigned int        truesize;
    /*atomic_t  users；这是个引用计数，表明了有多少实体引用了这个skb。
    其作用就是在销毁skb结构体时，先查看下users是否为零，
    若不为零，则调用函数递减下引用计数users即可；当某一次销毁时，users为零才真正释放内存空间。
    有两个操作函数：atomic_inc()引用计数增加1；atomic_dec()引用计数减去1；*/
    atomic_t        users;
};

/*在数据区的末端添加某协议的尾部*/
static inline unsigned char *__skb_put(struct sk_buff *skb, unsigned int len)
{
    unsigned char *tmp = skb_tail_pointer(skb);//获取当前skb->tail
    SKB_LINEAR_ASSERT(skb);
    skb->tail += len;
    skb->len  += len;
    return tmp;
}

/*把data指针增加n，相当于弹出数据*/
unsigned char *skb_pull(struct sk_buff *skb, unsigned int len);
static inline unsigned char *__skb_pull(struct sk_buff *skb, unsigned int len)
{
    skb->len -= len;
    BUG_ON(skb->len < skb->data_len);
    return skb->data += len;
}

/**
 *    skb_reserve - adjust headroom
 *    @skb: buffer to alter
 *    @len: bytes to move
 *
 *    Increase the headroom of an empty &sk_buff by reducing the tail
 *    room. This is only allowed for an empty buffer.
    给协议预留head的存储空间，只能对空的skb使用;
 */
static inline void skb_reserve(struct sk_buff *skb, int len)
{
    skb->data += len;
    skb->tail += len;
}

while(true){
         ServerSocket ss = new ServerSocket(8888);
         System.out.println("启动服务器....");
         Socket s = ss.accept();//阻塞点
         System.out.println("客户端:"+s.getInetAddress().getLocalHost()+"已连接到服务器");
         
         BufferedReader br = new BufferedReader(new InputStreamReader(s.getInputStream()));
         //读取客户端发送来的消息
         String mess = br.readLine();//阻塞2
         System.out.println("客户端："+mess);
         BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(s.getOutputStream()));
         bw.write(mess+"\n");
         bw.flush();  

}

struct eventpoll {
  ...
  /*红黑树的根节点，这棵树中存储着所有添加到epoll中的事件，
  也就是这个epoll监控的事件*/
  struct rb_root rbr;
  /*双向链表rdllist保存着将要通过epoll_wait返回给用户的、满足条件的事件*/
  struct list_head rdllist;
  ...
};

struct epitem {
  ...
  //红黑树节点
  struct rb_node rbn;
  //双向链表节点
  struct list_head rdllink;
  //事件句柄等信息
  struct epoll_filefd ffd;
  //指向其所属的eventepoll对象
  struct eventpoll *ep;
  //期待的事件类型
/* The structure that describe the interested events and the source fd */
  struct epoll_event event;
   //下一个epitem实例
   struct epitem *next;  ...
}; // 这里包含每一个事件对应着的信息。

struct epoll_event {
    __u32 events;
    __u64 data;
} EPOLL_PACKED;

struct __wait_queue {
    unsigned int        flags;
    void            *private;//等待队列的task_struct指针
    wait_queue_func_t    func;//进程唤醒后执行的回调函数
    struct list_head    task_list;
};

/*
 * Open an eventpoll file descriptor.
 */
SYSCALL_DEFINE1(epoll_create1, int, flags)
{
    int error, fd;
    struct eventpoll *ep = NULL;
    struct file *file;

    /* Check the EPOLL_* constant for consistency.  */
    BUILD_BUG_ON(EPOLL_CLOEXEC != O_CLOEXEC);

    if (flags & ~EPOLL_CLOEXEC)
        return -EINVAL;
    /*
     * Create the internal data structure ("struct eventpoll").
       创建一个eventpoll实例，里面初始化红黑树、链表的节点
     */
    error = ep_alloc(&ep);
    if (error < 0)
        return error;
    /*
     * Creates all the items needed to setup an eventpoll file. That is,
     * a file structure and a free file descriptor.
     */
    fd = get_unused_fd_flags(O_RDWR | (flags & O_CLOEXEC));
    if (fd < 0) {
        error = fd;
        goto out_free_ep;
    }
    file = anon_inode_getfile("[eventpoll]", &eventpoll_fops, ep,
                 O_RDWR | (flags & O_CLOEXEC));
    if (IS_ERR(file)) {
        error = PTR_ERR(file);
        goto out_free_fd;
    }
    ep->file = file;
    fd_install(fd, file);
    return fd;

out_free_fd:
    put_unused_fd(fd);
out_free_ep:
    ep_free(ep);
    return error;
}

/*生成eventpoll实例*/
static int ep_alloc(struct eventpoll **pep)
{
    int error;
    struct user_struct *user;
    struct eventpoll *ep;

    user = get_current_user();
    error = -ENOMEM;
    ep = kzalloc(sizeof(*ep), GFP_KERNEL);
    if (unlikely(!ep))
        goto free_uid;

    spin_lock_init(&ep->lock);
    mutex_init(&ep->mtx);
    init_waitqueue_head(&ep->wq);
    init_waitqueue_head(&ep->poll_wait);
    INIT_LIST_HEAD(&ep->rdllist);//readList第一个节点初始化
    ep->rbr = RB_ROOT;//红黑树根节点
    ep->ovflist = EP_UNACTIVE_PTR;
    ep->user = user;

    *pep = ep;

    return 0;

free_uid:
    free_uid(user);
    return error;
}

/*
 * The following function implements the controller interface for
 * the eventpoll file that enables the insertion/removal/change of
 * file descriptors inside the interest set.
 */
SYSCALL_DEFINE4(epoll_ctl, int, epfd, int, op, int, fd,
        struct epoll_event __user *, event)
{
    int error;
    int full_check = 0;
    struct fd f, tf;
    struct eventpoll *ep;
    struct epitem *epi;
    struct epoll_event epds;
    struct eventpoll *tep = NULL;
        ..........
        /*
     * Try to lookup the file inside our RB tree, Since we grabbed "mtx"
     * above, we can be sure to be able to use the item looked up by
     * ep_find() till we release the mutex. 在红黑树中根据fd找到epi结构体
     */
    epi = ep_find(ep, tf.file, fd);

    error = -EINVAL;
    switch (op) {
    case EPOLL_CTL_ADD:
        if (!epi) {
            epds.events |= POLLERR | POLLHUP;
            /*epoll节点插入，包括list节点和红黑树节点*/
            error = ep_insert(ep, &epds, tf.file, fd, full_check);
        } else
            error = -EEXIST;
        if (full_check)
            clear_tfile_check_list();
        break;
    case EPOLL_CTL_DEL:
        if (epi)
            /*epoll节点删除，包括list节点和红黑树节点*/
            error = ep_remove(ep, epi);
        else
            error = -ENOENT;
        break;
    case EPOLL_CTL_MOD:
        if (epi) {
            if (!(epi->event.events & EPOLLEXCLUSIVE)) {
                epds.events |= POLLERR | POLLHUP;
                /*epoll节点更改，包括list节点和红黑树节点*/
                error = ep_modify(ep, epi, &epds);
            }
        } else
            error = -ENOENT;
        break;
    }
        ..........
}

/*
 * Must be called with "mtx" held.
 */
static int ep_insert(struct eventpoll *ep, struct epoll_event *event,
             struct file *tfile, int fd, int full_check)
{
    int error, revents, pwake = 0;
    unsigned long flags;
    long user_watches;
    struct epitem *epi;
    struct ep_pqueue epq;

    user_watches = atomic_long_read(&ep->user->epoll_watches);
    if (unlikely(user_watches >= max_user_watches))
        return -ENOSPC;
    if (!(epi = kmem_cache_alloc(epi_cache, GFP_KERNEL)))
        return -ENOMEM;

    /* Item initialization follow here ... */
    INIT_LIST_HEAD(&epi->rdllink);//初始化readyList链表头
    INIT_LIST_HEAD(&epi->fllink);
    INIT_LIST_HEAD(&epi->pwqlist);
    epi->ep = ep;
    ep_set_ffd(&epi->ffd, tfile, fd);
    epi->event = *event;
    epi->nwait = 0;
    epi->next = EP_UNACTIVE_PTR;
    if (epi->event.events & EPOLLWAKEUP) {
        error = ep_create_wakeup_source(epi);
        if (error)
            goto error_create_wakeup_source;
    } else {
        RCU_INIT_POINTER(epi->ws, NULL);
    }

    /* Initialize the poll table using the queue callback */
    epq.epi = epi;
    /*注册回到函数*/
    init_poll_funcptr(&epq.pt, ep_ptable_queue_proc);

    /*
     * Attach the item to the poll hooks and get current event bits.
     * We can safely use the file* here because its usage count has
     * been increased by the caller of this function. Note that after
     * this operation completes, the poll callback can start hitting
     * the new item.
     */
    revents = ep_item_poll(epi, &epq.pt);

    /*
     * We have to check if something went wrong during the poll wait queue
     * install process. Namely an allocation for a wait queue failed due
     * high memory pressure.
     */
    error = -ENOMEM;
    if (epi->nwait < 0)
        goto error_unregister;

    /* Add the current item to the list of active epoll hook for this file */
    spin_lock(&tfile->f_lock);
    list_add_tail_rcu(&epi->fllink, &tfile->f_ep_links);
    spin_unlock(&tfile->f_lock);

    /*
     * Add the current item to the RB tree. All RB tree operations are
     * protected by "mtx", and ep_insert() is called with "mtx" held.
       epitem节点就是在这里插入红黑树的
     */
    ep_rbtree_insert(ep, epi);

    /* now check if we've created too many backpaths */
    error = -EINVAL;
    if (full_check && reverse_path_check())
        goto error_remove_epi;

    /* We have to drop the new item inside our item list to keep track of it */
    spin_lock_irqsave(&ep->lock, flags);

    /* If the file is already "ready" we drop it inside the ready list 
    */
    if ((revents & event->events) && !ep_is_linked(&epi->rdllink)) {
        list_add_tail(&epi->rdllink, &ep->rdllist);//把当前event加入readyList的尾部
        ep_pm_stay_awake(epi);

        /* Notify waiting tasks that events are available 
        如果网卡收到数据，需要唤醒等待的task，并执行实现设置好的回调函数
        */
        if (waitqueue_active(&ep->wq))
            wake_up_locked(&ep->wq);//唤醒进程后执行的回调函数
        if (waitqueue_active(&ep->poll_wait))
            pwake++;
    }

    spin_unlock_irqrestore(&ep->lock, flags);

    atomic_long_inc(&ep->user->epoll_watches);

    /* We have to call this outside the lock */
    if (pwake)
        ep_poll_safewake(&ep->poll_wait);

    return 0;
}

/*
 * Removes a "struct epitem" from the eventpoll RB tree and deallocates
 * all the associated resources. Must be called with "mtx" held.
 */
static int ep_remove(struct eventpoll *ep, struct epitem *epi)
{
    unsigned long flags;
    struct file *file = epi->ffd.file;

    /*
     * Removes poll wait queue hooks. We _have_ to do this without holding
     * the "ep->lock" otherwise a deadlock might occur. This because of the
     * sequence of the lock acquisition. Here we do "ep->lock" then the wait
     * queue head lock when unregistering the wait queue. The wakeup callback
     * will run by holding the wait queue head lock and will call our callback
     * that will try to get "ep->lock".
     */
    ep_unregister_pollwait(ep, epi);

    /* Remove the current item from the list of epoll hooks */
    spin_lock(&file->f_lock);
    list_del_rcu(&epi->fllink);
    spin_unlock(&file->f_lock);

    rb_erase(&epi->rbn, &ep->rbr);//从红黑树删除

    spin_lock_irqsave(&ep->lock, flags);
    if (ep_is_linked(&epi->rdllink))
        list_del_init(&epi->rdllink);//从readyList删除
    spin_unlock_irqrestore(&ep->lock, flags);

    wakeup_source_unregister(ep_wakeup_source(epi));
    /*
     * At this point it is safe to free the eventpoll item. Use the union
     * field epi->rcu, since we are trying to minimize the size of
     * 'struct epitem'. The 'rbn' field is no longer in use. Protected by
     * ep->mtx. The rcu read side, reverse_path_check_proc(), does not make
     * use of the rbn field.
     */
    call_rcu(&epi->rcu, epi_rcu_free);

    atomic_long_dec(&ep->user->epoll_watches);

    return 0;
}

/*
 * Implement the event wait interface for the eventpoll file. It is the kernel
 * part of the user space epoll_wait(2).
 */
SYSCALL_DEFINE4(epoll_wait, int, epfd, struct epoll_event __user *, events,
        int, maxevents, int, timeout)
{
    int error;
    struct fd f;
    struct eventpoll *ep;
        ..........
     /*
     * At this point it is safe to assume that the "private_data" contains
     * our own data structure.
     */
    ep = f.file->private_data;

    /* Time to fish for events ... */
    error = ep_poll(ep, events, maxevents, timeout);

}

/**
 * ep_poll - Retrieves ready events, and delivers them to the caller supplied
 *           event buffer.
 *           找到ready的事件
 * @ep: Pointer to the eventpoll context.
 * @events: Pointer to the userspace buffer where the ready events should be
 *          stored.
 * @maxevents: Size (in terms of number of events) of the caller event buffer.
 * @timeout: Maximum timeout for the ready events fetch operation, in
 *           milliseconds. If the @timeout is zero, the function will not block,
 *           while if the @timeout is less than zero, the function will block
 *           until at least one event has been retrieved (or an error
 *           occurred).
 *
 * Returns: Returns the number of ready events which have been fetched, or an
 *          error code, in case of error.
 */
static int ep_poll(struct eventpoll *ep, struct epoll_event __user *events,
           int maxevents, long timeout)
{
    int res = 0, eavail, timed_out = 0;
    unsigned long flags;
    u64 slack = 0;
    wait_queue_t wait;
    ktime_t expires, *to = NULL;

    if (timeout > 0) {
        struct timespec64 end_time = ep_set_mstimeout(timeout);

        slack = select_estimate_accuracy(&end_time);
        to = &expires;
        *to = timespec64_to_ktime(end_time);
    } else if (timeout == 0) {
        /*
         * Avoid the unnecessary trip to the wait queue loop, if the
         * caller specified a non blocking operation.
         */
        timed_out = 1;
        spin_lock_irqsave(&ep->lock, flags);
        goto check_events;
    }

fetch_events:
    spin_lock_irqsave(&ep->lock, flags);

    if (!ep_events_available(ep)) {
        /* 如果没有任何事件发生，就让出cpu；一旦有事件到达，会被ep_poll_callback函数唤醒！
         * We don't have any available event to return to the caller.
         * We need to sleep here, and we will be wake up by
         * ep_poll_callback() when events will become available.
         */
        init_waitqueue_entry(&wait, current);//把当前进程放入等待队列，并注册唤醒回调函数
        __add_wait_queue_exclusive(&ep->wq, &wait);

        for (;;) {
            /*
             * We don't want to sleep if the ep_poll_callback() sends us
             * a wakeup in between. That's why we set the task state
             * to TASK_INTERRUPTIBLE before doing the checks.
             */
            set_current_state(TASK_INTERRUPTIBLE);
            if (ep_events_available(ep) || timed_out)//如果有ready的事件，或已经等待超时就跳出死循环
                break;
            if (signal_pending(current)) {
                res = -EINTR;
                break;
            }

            spin_unlock_irqrestore(&ep->lock, flags);            //主动让出cpu，进入随眠状态；等有事件发生时，进程切换回来继续执行for循环，到上面的break代码跳出
            if (!schedule_hrtimeout_range(to, slack, HRTIMER_MODE_ABS))
                timed_out = 1;

            spin_lock_irqsave(&ep->lock, flags);
        }

        __remove_wait_queue(&ep->wq, &wait);//移除wait_queue
        __set_current_state(TASK_RUNNING);//唤醒后进程状态设置为running
    }
check_events:
    /* Is it worth to try to dig for events ? */
    eavail = ep_events_available(ep);//遍历readyList，发现不为空哦，说明有事件产生了！

    spin_unlock_irqrestore(&ep->lock, flags);

    /*
     * Try to transfer events to user space. In case we get 0 events and
     * there's still timeout left over, we go trying again in search of
     * more luck. 把ready的事件拷贝到用户空间，让上层应用继续处理
     */
    if (!res && eavail &&
        !(res = ep_send_events(ep, events, maxevents)) && !timed_out)
        goto fetch_events;

    return res;
}

/*
 * The core wakeup function. Non-exclusive wakeups (nr_exclusive == 0) just
 * wake everything up. If it's an exclusive wakeup (nr_exclusive == small +ve
 * number) then we wake all the non-exclusive tasks and one exclusive task.
 *
 * There are circumstances in which we can try to wake a task which has already
 * started to run but is not in state TASK_RUNNING. try_to_wake_up() returns
 * zero in this (rare) case, and we handle it by continuing to scan the queue.
   执行等待队列中每个task被唤醒后的回调函数
 */
static void __wake_up_common(wait_queue_head_t *q, unsigned int mode,
            int nr_exclusive, int wake_flags, void *key)
{
    wait_queue_t *curr, *next;

    list_for_each_entry_safe(curr, next, &q->task_list, task_list) {
        unsigned flags = curr->flags;
        /*执行回调函数*/
        if (curr->func(curr, mode, wake_flags, key) &&
                (flags & WQ_FLAG_EXCLUSIVE) && !--nr_exclusive)
            break;
    }
}

#include <iostream>  
#include <sys/socket.h>  
#include <sys/epoll.h>  
#include <netinet/in.h>  
#include <arpa/inet.h>  
#include <fcntl.h>  
#include <unistd.h>  
#include <stdio.h>  
#include <errno.h>  
  
using namespace std;  
  
#define MAXLINE 100  
#define OPEN_MAX 100  
#define LISTENQ 20  
#define SERV_PORT 5000  
#define INFTIM 1000  
  
void setnonblocking(int sock)  
{  
    int opts;  
     opts=fcntl(sock,F_GETFL);  
    if(opts<0)  
    {  
        perror("fcntl(sock,GETFL)");  
        exit(1);  
    }  
     opts = opts|O_NONBLOCK;  
    if(fcntl(sock,F_SETFL,opts)<0)  
    {  
        perror("fcntl(sock,SETFL,opts)");  
        exit(1);  
    }  
}  
  
int main(int argc, char* argv[])  
{  
    int i, maxi, listenfd, connfd, sockfd,epfd,nfds, portnumber;  
    ssize_t n;  
    char line[MAXLINE];  
    socklen_t clilen;  
    string szTemp("");  
  
    if ( 2 == argc )  
    {  
        if( (portnumber = atoi(argv[1])) < 0 )  
        {  
            fprintf(stderr,"Usage:%s portnumber\a\n",argv[0]);  
            return 1;  
        }  
    }  
    else  
    {  
        fprintf(stderr,"Usage:%s portnumber\a\n",argv[0]);  
        return 1;  
    }  
  
    //声明epoll_event结构体的变量,ev用于注册事件,数组用于回传要处理的事件  
    struct epoll_event ev, events[20];  
      
    //创建一个epoll的句柄，size用来告诉内核这个监听的数目一共有多大  
    epfd = epoll_create(256); //生成用于处理accept的epoll专用的文件描述符  
      
    struct sockaddr_in clientaddr;  
    struct sockaddr_in serveraddr;  
    listenfd = socket(AF_INET, SOCK_STREAM, 0);  
      
    //把socket设置为非阻塞方式  
    //setnonblocking(listenfd);  
  
    //设置与要处理的事件相关的文件描述符  
    //要监听的fd
    ev.data.fd=listenfd;  
      
    //设置要处理的事件类型  
    //要监听的事件：收到数据后可读、边缘触发
    ev.events=EPOLLIN|EPOLLET;  
  
    //注册epoll事件 ，也就上述要是监听fd+监听的事件，通过ev传入
    //本质就是在红黑树上增删改查
    epoll_ctl(epfd,EPOLL_CTL_ADD,listenfd,&ev);  
      
    bzero(&serveraddr, sizeof(serveraddr)); /*配置Server socket的相关信息 */  
    serveraddr.sin_family = AF_INET;  
    char *local_addr="127.0.0.1";  
    inet_aton(local_addr,&(serveraddr.sin_addr));//htons(portnumber);  
    serveraddr.sin_port=htons(portnumber);  
    bind(listenfd,(sockaddr *)&serveraddr, sizeof(serveraddr));  
    listen(listenfd, LISTENQ);  
      
    maxi = 0;  
      
    for ( ; ; ) {  
          
        //等待epoll事件的发生  
        //返回需要处理的事件数目nfds，如返回0表示已超时。  
        nfds=epoll_wait(epfd,events,20,500);  
          
        //处理所发生的所有事件  
        for(i=0; i < nfds; ++i)  
        {  
            //如果新监测到一个SOCKET用户连接到了绑定的SOCKET端口，建立新的连接。  
            if(events[i].data.fd == listenfd)  
            {  
                //收到了用户建立连接的请求再调用accept，避免阻塞浪费时间
                connfd = accept(listenfd,(sockaddr *)&clientaddr, &clilen);  
                if(connfd < 0)  
                {  
                    perror("connfd < 0");  
                    exit(1);  
                }  
                //setnonblocking(connfd);  
                char *str = inet_ntoa(clientaddr.sin_addr);  
                cout << "accapt a connection from " << str << endl;  
                  
                //设置用于读操作的文件描述符  
                  ev.data.fd=connfd;  
                  
                //设置用于注册的读操作事件  
                  ev.events=EPOLLIN|EPOLLET;  
  
                //注册ev  
                epoll_ctl(epfd,EPOLL_CTL_ADD,connfd,&ev); /* 添加新建立的fd，同样需要监控这个fd有没有收发数据、边缘触发等 */  
            }  
            //如果是已经连接的用户，并且收到数据，那么进行读入。  
            else if(events[i].events&EPOLLIN)  
            {  
                cout << "EPOLLIN" << endl;  
                if ( (sockfd = events[i].data.fd) < 0)  
                    continue;  
                //网卡收到了数据再调用recv把数据拷贝到用户空间，避免阻塞浪费时间
                if ( (n = recv(sockfd, line, sizeof(line), 0)) < 0)   
                {    
                    // Connection Reset:你连接的那一端已经断开了，而你却还试着在对方已断开的socketfd上读写数据！  
                    if (errno == ECONNRESET)  
                    {  
                        close(sockfd);  
                        events[i].data.fd = -1;  
                    }   
                    else  
                        std::cout<<"readline error"<<std::endl;  
                }   
                else if (n == 0) //读入的数据为空  
                {  
                    close(sockfd);  
                    events[i].data.fd = -1;  
                }  
                  
                szTemp = "";  
                szTemp += line;  
                szTemp = szTemp.substr(0,szTemp.find('\r')); /* remove the enter key */  
                memset(line,0,100); /* clear the buffer */  
                //line[n] = '\0';  
                cout << "Readin: " << szTemp << endl;  
                  
                //设置用于写操作的文件描述符  
                ev.data.fd=sockfd;  
                  
                //设置用于注册的写操作事件  
                ev.events=EPOLLOUT|EPOLLET;  
                  
                //修改sockfd上要处理的事件为EPOLLOUT，即fd上可写了、可以发送数据了
                epoll_ctl(epfd,EPOLL_CTL_MOD,sockfd,&ev); /* 修改红黑树 */  
  
            }  
            else if(events[i].events&EPOLLOUT) // 如果有数据发送  
            {  
                sockfd = events[i].data.fd;  
                szTemp = "Server:" + szTemp + "\n";  
                send(sockfd, szTemp.c_str(), szTemp.size(), 0); 
                  
                //设置用于读操作的文件描述符  
                ev.data.fd=sockfd;  
                 
                //设置用于注册的读操作事件  
                ev.events=EPOLLIN|EPOLLET;  
                  
                //修改sockfd上要处理的事件为EPOLIN ，即fd上可读了、可以接收数据了
                epoll_ctl(epfd,EPOLL_CTL_MOD,sockfd,&ev); /* 修改红黑树 */  
            }  
        } //(over)处理所发生的所有事件  
    } //(over)等待epoll事件的发生  
      
    close(epfd);  
    return 0;  
}

struct sched_class {
    const struct sched_class *next;
    /*
    1、全是成员函数：这里用函数指针来表达；
    2、调度的方式有很多种，比如cfs、rt、idle、deadline，每种方式的实现方法肯定不同，这里提供接口函数让不同的调度方式各自去实现（类似驱动的struct file_operations *ops结构体）
    */
    void (*enqueue_task) (struct rq *rq, struct task_struct *p, int flags);/*任务加入队列；cfs就是在红黑树插入节点*/
    void (*dequeue_task) (struct rq *rq, struct task_struct *p, int flags);/*任务移除队列；cfs就是在红黑树删除节点*/
    void (*yield_task) (struct rq *rq);/*让出任务*/
    bool (*yield_to_task) (struct rq *rq, struct task_struct *p, bool preempt);/*让出到任务*/

    void (*check_preempt_curr) (struct rq *rq, struct task_struct *p, int flags);

    /*
     * It is the responsibility of the pick_next_task() method that will
     * return the next task to call put_prev_task() on the @prev task or
     * something equivalent.
     *
     * May return RETRY_TASK when it finds a higher prio class has runnable
     * tasks.
     */
    struct task_struct * (*pick_next_task) (struct rq *rq,
                        struct task_struct *prev,
                        struct pin_cookie cookie);
    void (*put_prev_task) (struct rq *rq, struct task_struct *p);

#ifdef CONFIG_SMP
    int  (*select_task_rq)(struct task_struct *p, int task_cpu, int sd_flag, int flags);
    void (*migrate_task_rq)(struct task_struct *p);

    void (*task_woken) (struct rq *this_rq, struct task_struct *task);

    void (*set_cpus_allowed)(struct task_struct *p,
                 const struct cpumask *newmask);

    void (*rq_online)(struct rq *rq);
    void (*rq_offline)(struct rq *rq);
#endif

    void (*set_curr_task) (struct rq *rq);
    void (*task_tick) (struct rq *rq, struct task_struct *p, int queued);
    void (*task_fork) (struct task_struct *p);
    void (*task_dead) (struct task_struct *p);

    /*
     * The switched_from() call is allowed to drop rq->lock, therefore we
     * cannot assume the switched_from/switched_to pair is serliazed by
     * rq->lock. They are however serialized by p->pi_lock.
     */
    void (*switched_from) (struct rq *this_rq, struct task_struct *task);
    void (*switched_to) (struct rq *this_rq, struct task_struct *task);
    void (*prio_changed) (struct rq *this_rq, struct task_struct *task,
                 int oldprio);

    unsigned int (*get_rr_interval) (struct rq *rq,
                     struct task_struct *task);

    void (*update_curr) (struct rq *rq);

#define TASK_SET_GROUP  0
#define TASK_MOVE_GROUP    1

#ifdef CONFIG_FAIR_GROUP_SCHED
    void (*task_change_group) (struct task_struct *p, int type);
#endif
};

#define MAX_USER_RT_PRIO    100
#define MAX_RT_PRIO         MAX_USER_RT_PRIO
#define MAX_PRIO            (MAX_RT_PRIO + 40)//140个优先级（0 ~ 139，数值越小优先级越高）

#define BITMAP_SIZE ((((MAX_PRIO+1+7)/8)+sizeof(long)-1)/sizeof(long))

struct prio_array {
    int nr_active;//所有优先级队列中的总任务数。
    unsigned long bitmap[BITMAP_SIZE];//每个位对应一个优先级的任务队列，用于记录哪个任务队列不为空，能通过 bitmap 够快速找到不为空的任务队列
    struct list_head queue[MAX_PRIO];//优先级队列数组，每个元素维护一个优先级队列，比如索引为0的元素维护着优先级为0的任务队列
};

/*
 * Nice levels are multiplicative, with a gentle 10% change for every
 * nice level changed. I.e. when a CPU-bound task goes from nice 0 to
 * nice 1, it will get ~10% less CPU time than another CPU-bound task
 * that remained on nice 0.
 *
 * The "10% effect" is relative and cumulative: from _any_ nice level,
 * if you go up 1 level, it's -10% CPU usage, if you go down 1 level
 * it's +10% CPU usage. (to achieve that we use a multiplier of 1.25.
 * If a task goes up by ~10% and another task goes down by ~10% then
 * the relative distance between them is ~25%.)
 */
const int sched_prio_to_weight[40] = {
 /* -20 */     88761,     71755,     56483,     46273,     36291,
 /* -15 */     29154,     23254,     18705,     14949,     11916,
 /* -10 */      9548,      7620,      6100,      4904,      3906,
 /*  -5 */      3121,      2501,      1991,      1586,      1277,
 /*   0 */      1024,       820,       655,       526,       423,
 /*   5 */       335,       272,       215,       172,       137,
 /*  10 */       110,        87,        70,        56,        45,
 /*  15 */        36,        29,        23,        18,        15,
};

struct task_struct {
    .......
        int prio, static_prio, normal_prio;
    unsigned int rt_priority;
    const struct sched_class *sched_class;/*调度策略的实例*/
    struct sched_entity se;/*cfs调度策略，包含了rb_node*/
    struct sched_rt_entity rt;/*real time调度策略*/
#ifdef CONFIG_CGROUP_SCHED
    struct task_group *sched_task_group;
#endif
    struct sched_dl_entity dl;/*deadline 调度*/
    .......
}

struct sched_entity {/*cfs调度策略*/
    struct load_weight    load;        /* for load-balancing */
    struct rb_node        run_node;   /*调度实体是由红黑树组织起来的*/ 
    struct list_head    group_node;
    unsigned int        on_rq;
    /*构造红黑树时，其实下面的每一项都可以用作节点的key；
    1、但是这里选vruntime作为key构造红黑树，换句话说用vruntime来排序，小的靠左，大的靠右
    2、如果不同进程的vruntime一样，可以加很小的数改成不一样的
    */
    u64            exec_start;
    u64            sum_exec_runtime;
    u64            vruntime;/*红黑树节点排序的变量*/
    u64            prev_sum_exec_runtime;

    u64            nr_migrations;

#ifdef CONFIG_SCHEDSTATS
    struct sched_statistics statistics;
#endif

#ifdef CONFIG_FAIR_GROUP_SCHED
    int            depth;
    struct sched_entity    *parent;
    /* rq on which this entity is (to be) queued: */
    struct cfs_rq        *cfs_rq;
    /* rq "owned" by this entity/group: */
    struct cfs_rq        *my_q;
#endif

#ifdef CONFIG_SMP
    /*
     * Per entity load average tracking.
     *
     * Put into separate cache line so it does not
     * collide with read-mostly values above.
     */
    struct sched_avg    avg ____cacheline_aligned_in_smp;
#endif
};

/* CFS-related fields in a runqueue */
struct cfs_rq {
......
    struct rb_root tasks_timeline;/*红黑树的root根节点*/
    struct rb_node *rb_leftmost;/*红黑树最左边的节点，也就是vruntime最小的节点*/

    /*
     * 'curr' points to currently running entity on this cfs_rq.
     * It is set to NULL otherwise (i.e when none are currently running).
     */
    struct sched_entity *curr, *next, *last, *skip;
......

}

/*
 * Update the current task's runtime statistics.
 */
static void update_curr(struct cfs_rq *cfs_rq)
{
    struct sched_entity *curr = cfs_rq->curr;
    u64 now = rq_clock_task(rq_of(cfs_rq));
    u64 delta_exec;

    if (unlikely(!curr))
        return;
    /*计算进程已经执行的时间*/
    delta_exec = now - curr->exec_start;
    if (unlikely((s64)delta_exec <= 0))
        return;

    curr->exec_start = now;//更新开始执行的时间

    schedstat_set(curr->statistics.exec_max,
              max(delta_exec, curr->statistics.exec_max));

    curr->sum_exec_runtime += delta_exec;
    schedstat_add(cfs_rq->exec_clock, delta_exec);

    curr->vruntime += calc_delta_fair(delta_exec, curr);//更新vruntime
    update_min_vruntime(cfs_rq);

    if (entity_is_task(curr)) {
        struct task_struct *curtask = task_of(curr);

        trace_sched_stat_runtime(curtask, delta_exec, curr->vruntime);
        cpuacct_charge(curtask, delta_exec);
        account_group_exec_runtime(curtask, delta_exec);
    }

    account_cfs_rq_runtime(cfs_rq, delta_exec);
}

/*
 * Enqueue an entity into the rb-tree:
 */
static void __enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se)
{
    struct rb_node **link = &cfs_rq->tasks_timeline.rb_node;/*红黑树根节点*/
    struct rb_node *parent = NULL;
    struct sched_entity *entry;
    int leftmost = 1;

    /*
     * Find the right place in the rbtree:
     */
    while (*link) {
        parent = *link;
        /*找到节点实例的首地址，就是container_of的宏定义*/
        entry = rb_entry(parent, struct sched_entity, run_node);
        /*
         * We dont care about collisions. Nodes with
         * the same key stay together.
         */
        if (entity_before(se, entry)) {
            link = &parent->rb_left;
        } else {
            link = &parent->rb_right;
            leftmost = 0;
        }
    }

    /*
     * Maintain a cache of leftmost tree entries (it is frequently
     * used):
     */
    if (leftmost)
        cfs_rq->rb_leftmost = &se->run_node;
    /*在红黑树中插入节点，即将node节点插入到parent节点的左树或者右树，整个过程会动态调整树结构保持平衡；*/
    rb_link_node(&se->run_node, parent, link);
    /*设置节点的颜色*/
    rb_insert_color(&se->run_node, &cfs_rq->tasks_timeline);
}

static void __dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se)
{
    if (cfs_rq->rb_leftmost == &se->run_node) {
        struct rb_node *next_node;

        next_node = rb_next(&se->run_node);
        cfs_rq->rb_leftmost = next_node;
    }

    rb_erase(&se->run_node, &cfs_rq->tasks_timeline);
}

/*
 * Pick the next process, keeping these things in mind, in this order:
 * 1) keep things fair between processes/task groups
 * 2) pick the "next" process, since someone really wants that to run
 * 3) pick the "last" process, for cache locality
 * 4) do not run the "skip" process, if something else is available
 */
static struct sched_entity *
pick_next_entity(struct cfs_rq *cfs_rq, struct sched_entity *curr)
{
    struct sched_entity *left = __pick_first_entity(cfs_rq);//树上最左边的节点
    struct sched_entity *se;

    /*
     * If curr is set we have to see if its left of the leftmost entity
     * still in the tree, provided there was anything in the tree at all.
       若 second 为空, 或者 curr 的 vruntime 更小 
     */
    if (!left || (curr && entity_before(curr, left)))
        left = curr;

    se = left; /* ideally we run the leftmost entity */

    /*
     * Avoid running the skip buddy, if running something else can
     * be done without getting too unfair.
     */
    if (cfs_rq->skip == se) {
        struct sched_entity *second;

        if (se == curr) {
            second = __pick_first_entity(cfs_rq);/*返回最左边、也就是vruntime最小的节点*/
        } else {
            second = __pick_next_entity(se);/*找到比se节点大的第一个节点*/
            if (!second || (curr && entity_before(curr, second)))
                second = curr;
        }
        /*判断是否应该抢占当前进程*/
        if (second && wakeup_preempt_entity(second, left) < 1)
            se = second;
    }

    /*
     * Prefer last buddy, try to return the CPU to a preempted task.
     */
    if (cfs_rq->last && wakeup_preempt_entity(cfs_rq->last, left) < 1)
        se = cfs_rq->last;

    /*
     * Someone really wants this to run. If it's not unfair, run it.
     */
    if (cfs_rq->next && wakeup_preempt_entity(cfs_rq->next, left) < 1)
        se = cfs_rq->next;

    clear_buddies(cfs_rq, se);

    return se;
}

struct kprobe {
    struct hlist_node hlist：//被用于kprobe全局hash，索引值为被探测点的地址；
    struct list_head list：//用于链接同一被探测点的不同探测kprobe；
    kprobe_opcode_t *addr：//被探测点的地址；
    const char *symbol_name：//被探测函数的名字；
    unsigned int offset：//被探测点在函数内部的偏移，用于探测函数内部的指令，如果该值为0表示函数的入口；
    kprobe_pre_handler_t pre_handler：//在被探测点指令执行之前调用的回调函数；
    kprobe_post_handler_t post_handler：//在被探测指令执行之后调用的回调函数；
    kprobe_fault_handler_t fault_handler：//在执行pre_handler、post_handler或单步执行被探测指令时出现内存异常则会调用该回调函数；
    kprobe_break_handler_t break_handler：//在执行某一kprobe过程中触发了断点指令后会调用该函数，用于实现jprobe；
    kprobe_opcode_t opcode：//保存的被探测点原始指令；
    struct arch_specific_insn ainsn：//被复制的被探测点的原始指令，用于单步执行，架构强相关（可能包含指令模拟函数）；
    u32 flags：//状态标记。       
}

int register_kprobe(struct kprobe *kp)      //向内核注册kprobe探测点
void unregister_kprobe(struct kprobe *kp)   //卸载kprobe探测点
int register_kprobes(struct kprobe **kps, int num)     //注册探测函数向量，包含多个探测点
void unregister_kprobes(struct kprobe **kps, int num)  //卸载探测函数向量，包含多个探测点
int disable_kprobe(struct kprobe *kp)       //临时暂停指定探测点的探测
int enable_kprobe(struct kprobe *kp)        //恢复指定探测点的探测

static int __init do_kprobe_initcalls(void)
{
    int ret = 0;
    struct kprobe_initcall const *const *initcall_p;

    for (initcall_p = __start_kprobe_initcall;
         initcall_p < __stop_kprobe_initcall; initcall_p++) {
        struct kprobe_initcall const *initcall = *initcall_p;

        if (initcall->init) {
            ret = initcall->init();
            if (ret < 0)
                goto exit;
        }
    }

    return 0;
exit:
    while (--initcall_p >= __start_kprobe_initcall) {
        struct kprobe_initcall const *initcall = *initcall_p;

        if (initcall->exit)
            initcall->exit();
    }

    return ret;
}

static void do_kprobe_exitcalls(void)
{
    struct kprobe_initcall const *const *initcall_p =
            __stop_kprobe_initcall;

    while (--initcall_p >= __start_kprobe_initcall) {
        struct kprobe_initcall const *initcall = *initcall_p;

        if (initcall->exit)
            initcall->exit();
    }
}

static int __init kprobes_init(void)
{
    int ret;
    ret = do_kprobe_initcalls();
    if (ret < 0)
        return ret;

    return ret;
}

static void __exit kprobes_exit(void)
{
    do_kprobe_exitcalls();
}

module_init(kprobes_init);
module_exit(kprobes_exit);

struct kretprobe execve_kretprobe = {
        .kp.symbol_name = P_GET_SYSCALL_NAME(execve),
        .entry_handler = execve_entry_handler,
        .data_size = sizeof(struct execve_data),
        .handler = execve_handler,
};
struct kprobe call_usermodehelper_exec_kprobe = {
        .symbol_name = "call_usermodehelper_exec",
        .pre_handler = call_usermodehelper_exec_pre_handler,
};

struct kprobe rename_kprobe = {
        .symbol_name = P_GET_SYSCALL_NAME(rename),
        .pre_handler = rename_pre_handler,
};

struct kprobe renameat_kprobe = {
        .symbol_name = P_GET_SYSCALL_NAME(renameat),
        .pre_handler = renameat_pre_handler,
};

#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,15,0)
struct kprobe renameat2_kprobe = {
        .symbol_name = P_GET_SYSCALL_NAME(renameat2),
        .pre_handler = renameat_pre_handler,
};
#endif

struct kprobe link_kprobe = {
        .symbol_name = P_GET_SYSCALL_NAME(link),
        .pre_handler = link_pre_handler,
};

struct kprobe linkat_kprobe = {
        .symbol_name = P_GET_SYSCALL_NAME(linkat),
        .pre_handler = linkat_pre_handler,
};

struct kprobe ptrace_kprobe = {
        .symbol_name = P_GET_SYSCALL_NAME(ptrace),
        .pre_handler = ptrace_pre_handler,
};

struct kretprobe udp_recvmsg_kretprobe = {
        .kp.symbol_name = "udp_recvmsg",
        .data_size = sizeof(struct udp_recvmsg_data),
        .handler = udp_recvmsg_handler,
        .entry_handler = udp_recvmsg_entry_handler,
};

#if IS_ENABLED(CONFIG_IPV6)
struct kretprobe udpv6_recvmsg_kretprobe = {
        .kp.symbol_name = "udpv6_recvmsg",
        .data_size = sizeof(struct udp_recvmsg_data),
        .handler = udp_recvmsg_handler,
        .entry_handler = udpv6_recvmsg_entry_handler,
};

struct kretprobe ip6_datagram_connect_kretprobe = {
        .kp.symbol_name = "ip6_datagram_connect",
        .data_size = sizeof(struct connect_data),
        .handler = connect_handler,
        .entry_handler = ip6_datagram_connect_entry_handler,
};

struct kretprobe tcp_v6_connect_kretprobe = {
        .kp.symbol_name = "tcp_v6_connect",
        .data_size = sizeof(struct connect_data),
        .handler = connect_handler,
        .entry_handler = tcp_v6_connect_entry_handler,
};
#endif

struct kretprobe ip4_datagram_connect_kretprobe = {
        .kp.symbol_name = "ip4_datagram_connect",
        .data_size = sizeof(struct connect_data),
        .handler = connect_handler,
        .entry_handler = ip4_datagram_connect_entry_handler,
};

struct kretprobe tcp_v4_connect_kretprobe = {
        .kp.symbol_name = "tcp_v4_connect",
        .data_size = sizeof(struct connect_data),
        .handler = connect_handler,
        .entry_handler = tcp_v4_connect_entry_handler,
};

struct kretprobe connect_syscall_kretprobe = {
        .kp.symbol_name = P_GET_SYSCALL_NAME(connect),
        .data_size = sizeof(struct connect_syscall_data),
        .handler = connect_syscall_handler,
        .entry_handler = connect_syscall_entry_handler,
};

struct kretprobe accept_kretprobe = {
        .kp.symbol_name = P_GET_SYSCALL_NAME(accept),
        .data_size = sizeof(struct accept_data),
        .handler = accept_handler,
        .entry_handler = accept_entry_handler,
};

struct kretprobe accept4_kretprobe = {
        .kp.symbol_name = P_GET_SYSCALL_NAME(accept4),
        .data_size = sizeof(struct accept_data),
        .handler = accept_handler,
        .entry_handler = accept4_entry_handler,
};

struct kprobe do_init_module_kprobe = {
        .symbol_name = "do_init_module",
        .pre_handler = do_init_module_pre_handler,
};

struct kretprobe update_cred_kretprobe = {
        .kp.symbol_name = "commit_creds",
        .data_size = sizeof(struct update_cred_data),
        .handler = update_cred_handler,
        .entry_handler = update_cred_entry_handler,
};

struct kprobe security_inode_create_kprobe = {
        .symbol_name = "security_inode_create",
        .pre_handler = security_inode_create_pre_handler,
};

struct kretprobe bind_kretprobe = {
        .kp.symbol_name = P_GET_SYSCALL_NAME(bind),
        .data_size = sizeof(struct bind_data),
        .handler = bind_handler,
        .entry_handler = bind_entry_handler,
};

struct kprobe mprotect_kprobe = {
        .symbol_name = "security_file_mprotect",
        .pre_handler = mprotect_pre_handler,
};

struct kprobe setsid_kprobe = {
        .symbol_name = P_GET_SYSCALL_NAME(setsid),
        .pre_handler = setsid_pre_handler,
};

#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 17, 0)
struct kprobe memfd_create_kprobe = {
        .symbol_name = P_GET_SYSCALL_NAME(memfd_create),
        .pre_handler = memfd_create_kprobe_pre_handler,
};
#endif

struct kprobe prctl_kprobe = {
        .symbol_name = P_GET_SYSCALL_NAME(prctl),
        .pre_handler = prctl_pre_handler,
};

struct kprobe open_kprobe = {
        .symbol_name = P_GET_SYSCALL_NAME(open),
        .pre_handler = open_pre_handler,
};

struct kprobe kill_kprobe = {
        .symbol_name = P_GET_SYSCALL_NAME(kill),
        .pre_handler = kill_pre_handler,
};

struct kprobe tkill_kprobe = {
        .symbol_name = P_GET_SYSCALL_NAME(tkill),
        .pre_handler = tkill_pre_handler,
};

struct kprobe nanosleep_kprobe = {
        .symbol_name = P_GET_SYSCALL_NAME(nanosleep),
        .pre_handler = nanosleep_pre_handler,
};

struct kprobe exit_kprobe = {
        .symbol_name = P_GET_SYSCALL_NAME(exit),
        .pre_handler = exit_pre_handler,
};

struct kprobe exit_group_kprobe = {
        .symbol_name = P_GET_SYSCALL_NAME(exit_group),
        .pre_handler = exit_group_pre_handler,
};

struct kprobe security_path_rmdir_kprobe = {
        .symbol_name = "security_path_rmdir",
        .pre_handler = security_path_rmdir_pre_handler,
};

struct kprobe security_path_unlink_kprobe = {
        .symbol_name = "security_path_unlink",
        .pre_handler = security_path_unlink_pre_handler,
};

int execve_handler(struct kretprobe_instance *ri, struct pt_regs *regs)
{
    int sa_family = -1;
    int dport = 0, sport = 0;

    __be32 dip4;
    __be32 sip4;
    pid_t socket_pid = -1;

    char *pname = DEFAULT_RET_STR;
    char *tmp_stdin = DEFAULT_RET_STR;
    char *tmp_stdout = DEFAULT_RET_STR;
    char *buffer = NULL;
    char *pname_buf = NULL;
    char *pid_tree = NULL;
    char *socket_pname = "-1";
    char *socket_pname_buf = NULL;
    char *tty_name = "-1";
    char *exe_path = DEFAULT_RET_STR;
    char *pgid_exe_path = "-1";
    char *stdin_buf = NULL;
    char *stdout_buf = NULL;

    struct in6_addr dip6;
    struct in6_addr sip6;
    struct file *file;
    struct execve_data *data;
    struct tty_struct *tty;

    data = (struct execve_data *)ri->data;
    buffer = kzalloc(PATH_MAX, GFP_ATOMIC);
    /*当前进程所执行文件的路径*/
    exe_path = smith_get_exe_file(buffer, PATH_MAX);

    tty = get_current_tty();//得到当前终端
    if(tty && strlen(tty->name) > 0)
        tty_name = tty->name;

    //exe filter check and argv filter check
    if (execve_exe_check(exe_path) || execve_argv_check(data->argv))
        goto out;
    /*得到当前进程的socket，从这里也可以看出是不是中了木马的反弹webshell*/
    get_process_socket(&sip4, &sip6, &sport, &dip4, &dip6, &dport,
                       &socket_pname, &socket_pname_buf, &socket_pid,
                       &sa_family);

    //if socket exist,get pid tree
    if (sa_family == AF_INET6 || sa_family == AF_INET)
        pid_tree = smith_get_pid_tree(PID_TREE_LIMIT);
    else
        pid_tree = smith_get_pid_tree(PID_TREE_LIMIT_LOW);

    // get stdin
    file = fget_raw(0);
    if (file) {
        stdin_buf = kzalloc(256, GFP_ATOMIC);
        tmp_stdin = smith_d_path(&(file->f_path), stdin_buf, 256);
        fput(file);
    }

    //get stdout
    file = fget_raw(1);
    if (file) {
        stdout_buf = kzalloc(256, GFP_ATOMIC);
        tmp_stdout = smith_d_path(&(file->f_path), stdout_buf, 256);
        fput(file);
    }

    pname_buf = kzalloc(PATH_MAX, GFP_ATOMIC);
    pname = smith_d_path(&current->fs->pwd, pname_buf, PATH_MAX);
    /*打印收集到的各种数据*/
    if (sa_family == AF_INET) {
        execve_print(pname,
                     exe_path, pgid_exe_path, data->argv,
                     tmp_stdin, tmp_stdout,
                     dip4, dport, sip4, sport,
                     pid_tree, tty_name, socket_pid, socket_pname,
                     data->ssh_connection, data->ld_preload,
                     regs_return_value(regs));
    }
#if IS_ENABLED(CONFIG_IPV6)
        else if (sa_family == AF_INET6) {
        execve6_print(pname,
                  exe_path, pgid_exe_path, data->argv,
                  tmp_stdin, tmp_stdout,
                  &dip6, dport, &sip6, sport,
                  pid_tree, tty_name, socket_pid, socket_pname,
                  data->ssh_connection, data->ld_preload,
                  regs_return_value(regs));
    }
#endif
    else {
        execve_nosocket_print(pname,
                              exe_path, pgid_exe_path, data->argv,
                              tmp_stdin, tmp_stdout,
                              pid_tree, tty_name,
                              data->ssh_connection, data->ld_preload,
                              regs_return_value(regs));
    }

out:
    if (pname_buf)
        kfree(pname_buf);

    if (stdin_buf)
        kfree(stdin_buf);

    if (stdout_buf)
        kfree(stdout_buf);

    if (buffer)
        kfree(buffer);

    if (data->free_argv)
        kfree(data->argv);

    if (pid_tree)
        kfree(pid_tree);

    if (data->free_ld_preload)
        kfree(data->ld_preload);

    if (data->free_ssh_connection)
        kfree(data->ssh_connection);

    if(tty)
        tty_kref_put(tty);

    return 0;
}

SYSCALL_DEFINE3(mprotect, .., start, .., len, .., prot)
    -> do_mprotect_pkey(start, len, prot, pkey=-1)
        -> mprotect_fixup(vma, .., start, end, newflags)
            -> change_protection(vma, start, end, newprot, cp_flags)
                -> change_protection_range(vma, addr, end, newprot, cp_flags)
                    -> change_p4d_range(vma, pgd, add, end, newprot, cp_flags)
                        -> change_pmd_range(vma, pud, addr, end, newprot, cp_flags)
                            -> change_pte_range(vma, pmd, addr, end, newprot, cp_flags)

int mprotect_pre_handler(struct kprobe *p, struct pt_regs *regs)
{
    int target_pid = -1;
    unsigned long prot;

    char *file_path = "-1";
    char *file_buf = NULL;
    char *vm_file_path = "-1";
    char *vm_file_buff = NULL;
    char *exe_path = "-1";
    char *abs_buf = NULL;
    char *pid_tree = NULL;

    struct vm_area_struct *vma;

    //only get PROT_EXEC mprotect info
    //The memory can be used to store instructions which can then be executed. On most architectures,
    //this flag implies that the memory can be read (as if PROT_READ had been specified).
    prot = (unsigned long)p_regs_get_arg2(regs);
    if (prot & PROT_EXEC) {
        abs_buf = kzalloc(PATH_MAX, GFP_ATOMIC);
        exe_path = smith_get_exe_file(abs_buf, PATH_MAX);//当前进程对应文件的路径

        vma = (struct vm_area_struct *)p_regs_get_arg1(regs);
        if (IS_ERR_OR_NULL(vma)) {
            mprotect_print(exe_path, prot, "-1", -1, "-1", "-1");
        } else {
            rcu_read_lock();
            if (!IS_ERR_OR_NULL(vma->vm_mm)) {
                if (!IS_ERR_OR_NULL(&vma->vm_mm->exe_file)) {
                    if (get_file_rcu(vma->vm_mm->exe_file)) {
                        file_buf = kzalloc(PATH_MAX, GFP_ATOMIC);
                        file_path = smith_d_path(&vma->vm_mm->exe_file->f_path, file_buf, PATH_MAX);
                        fput(vma->vm_mm->exe_file);
                    }
                }
#ifdef CONFIG_MEMCG
                target_pid = vma->vm_mm->owner->pid;
#endif
            }

            if (!IS_ERR_OR_NULL(vma->vm_file)) {
                if (get_file_rcu(vma->vm_file)) {
                    vm_file_buff =
                            kzalloc(PATH_MAX, GFP_ATOMIC);
                    vm_file_path = smith_d_path(&vma->vm_file->f_path, vm_file_buff, PATH_MAX);
                    fput(vma->vm_file);
                }
            }
            rcu_read_unlock();
            /*被修改内存属性的pid树，从这里可以看到关键进程是不是在被调试等*/
            pid_tree = smith_get_pid_tree(PID_TREE_LIMIT);
            mprotect_print(exe_path, prot, file_path, target_pid, vm_file_path, pid_tree);
        }

        if (pid_tree)
            kfree(pid_tree);

        if (file_buf)
            kfree(file_buf);

        if (abs_buf)
            kfree(abs_buf);

        if (vm_file_buff)
            kfree(vm_file_buff);
    }
    return 0;
}

int ptrace_pre_handler(struct kprobe *p, struct pt_regs *regs)
{
    long request;
    request = (long)p_get_arg1(regs);

    //only get PTRACE_POKETEXT/PTRACE_POKEDATA ptrace
    //Read a word at the address addr in the tracee's memory,
    //returning the word as the result of the ptrace() call.  Linux
    //does not have separate text and data address spaces, so these  linux居然没区分代码和数据空间
    //two requests are currently equivalent.  (data is ignored; but
    //see NOTES.)
    /*PTRACE_PEEKTEXT或PTRACE_PEEKDATA:从addr参数指示的地址开始读取一个WORD的长度的数据并通过返回值传递回来*/
    if (request == PTRACE_POKETEXT || request == PTRACE_POKEDATA) {
        long pid;
        void *addr;
        char *exe_path = DEFAULT_RET_STR;
        char *buffer = NULL;
        char *pid_tree = NULL;

        pid = (long)p_get_arg2(regs);
        addr = (void *)p_get_arg3(regs);

        if (IS_ERR_OR_NULL(addr))
            return 0;

        buffer = kzalloc(PATH_MAX, GFP_ATOMIC);
        /*得到当前文件路径*/
        exe_path = smith_get_exe_file(buffer, PATH_MAX);
        /*得到当前进程的进程树*/
        pid_tree = smith_get_pid_tree(PID_TREE_LIMIT);
        /*打印结果*/
        ptrace_print(request, pid, addr, "-1", exe_path, pid_tree);

        if(buffer)
            kfree(buffer);

        if(pid_tree)
            kfree(pid_tree);
    }

    return 0;
}

typedef struct {
                unsigned int slock;
        } raw_spinlock_t;#define __RAW_SPIN_LOCK_UNLOCKED { 1 }
static inline void __raw_spin_lock(raw_spinlock_t *lock)
       {
                asm volatile("\n1:\t"
                        LOCK_PREFIX " ; decb %0\n\t"
                // lock->slock减1
                        "jns 3f\n"
                //如果不为负.跳转到3f.3f后面没有任何指令，即为退出
                        "2:\t"
                        "rep;nop\n\t"
                //重复执行nop.nop是x86的小延迟函数
                        "cmpb $0,%0\n\t"
                        "jle 2b\n\t"
                //如果lock->slock不大于0，跳转到标号2，即继续重复执行nop
                        "jmp 1b\n"
                //如果lock->slock大于0，跳转到标号1，重新判断锁的slock成员
                        "3:\n\t"
                        : "+m" (lock->slock) : : "memory");
      }

static inline void __raw_spin_unlock(raw_spinlock_t *lock)
        {
                asm volatile("movb $1,%0" : "+m" (lock->slock) :: "memory");
        }

#if __LINUX_ARM_ARCH__ < 6
        #error SMP not supported on pre-ARMv6 CPUs //ARMv6后，才有多核ARM处理器
        #endif
        ……
        static inline void __raw_spin_lock(raw_spinlock_t *lock)
        {
                unsigned long tmp;
                __asm__ __volatile__(
        "1: ldrex        %0, [%1]\n"
        //取lock->lock放在 tmp里，并且设置&lock->lock这个内存地址为独占访问
        "        teq %0, #0\n"
        //测试lock_lock是否为0，影响标志位z
        #ifdef CONFIG_CPU_32v6K
        "        wfene\n"
        #endif
        "        strexeq %0, %2, [%1]\n"
        //如果lock_lock是0，并且是独占访问这个内存，就向lock->lock里写入1，并向tmp返回0，同时清除独占标记
        "        teqeq %0, #0\n"
        //如果lock_lock是0，并且strexeq返回了0，表示加锁成功，返回
        " bne 1b"
        //如果上面的条件(1：lock->lock里不为0，2：strexeq失败)有一个符合，就在原地打转
                : "=&r" (tmp) //%0：输出放在tmp里，可以是任意寄存器
                : "r" (&lock->lock), "r" (1)
        //%1：取&lock->lock放在任意寄存器，%2：任意寄存器放入1
                : "cc"); //状态寄存器可能会改变
                smp_mb();
        }

static inline void __raw_spin_unlock(raw_spinlock_t *lock)
        {
                smp_mb();
                __asm__ __volatile__(
        "         str %1, [%0]\n" // 向lock->lock里写0，解锁
        #ifdef CONFIG_CPU_32v6K
        "         mcr p15, 0, %1, c7, c10, 4\n"
        "         sev"
        #endif
                :
                : "r" (&lock->lock), "r" (0) //%0取&lock->lock放在任意寄存器，%1：任意寄存器放入0
                : "cc");
        }

public class Singleton {
    private volatile static Singleton uniqueSingleton;

    private Singleton() {
    }

    public Singleton getInstance() {
        if (null == uniqueSingleton) {
            synchronized (Singleton.class) {
                if (null == uniqueSingleton) {
                    uniqueSingleton = new Singleton();
                }
            }
        }
        return uniqueSingleton;
    }
}

inline jint     Atomic::add    (jint     add_value, volatile jint*     dest) {
  jint addend = add_value;
  int mp = os::is_MP();
  __asm__ volatile (  LOCK_IF_MP(%3) "xaddl %0,(%2)"
                    : "=r" (addend)
                    : "0" (addend), "r" (dest), "r" (mp)
                    : "cc", "memory");
  return addend + add_value;
}

inline jint     Atomic::cmpxchg    (jint     exchange_value, volatile jint*     dest, jint     compare_value) {
  int mp = os::is_MP();
  __asm__ volatile (LOCK_IF_MP(%4) "cmpxchgl %1,(%3)"
                    : "=a" (exchange_value)
                    : "r" (exchange_value), "a" (compare_value), "r" (dest), "r" (mp)
                    : "cc", "memory");
  return exchange_value;
}

struct free_area {
    struct list_head    free_list;
    unsigned long        nr_free;
};

/**
 * 内存管理区描述符
 */
struct zone {
    /* Fields commonly accessed by the page allocator */
    /**
     * 管理区中空闲页的数目
     */
    unsigned long        free_pages;
    /**
     * Pages_min-管理区中保留页的数目
     * Page_low-回收页框使用的下界。同时也被管理区分配器为作为阈值使用。
     * pages_high-回收页框使用的上界，同时也被管理区分配器作为阈值使用。
     */
    unsigned long        pages_min, pages_low, pages_high;
    /*
     * We don't know if the memory that we're going to allocate will be freeable
     * or/and it will be released eventually, so to avoid totally wasting several
     * GB of ram we must reserve some of the lower zone memory (otherwise we risk
     * to run OOM on the lower zones despite there's tons of freeable ram
     * on the higher zones). This array is recalculated at runtime if the
     * sysctl_lowmem_reserve_ratio sysctl changes.
     */
    /**
     * 为内存不足保留的页框，分别为各种内存域指定了若干页
     * 用于一些无论如何都不能失败的关键性内存分配
     */
    unsigned long        lowmem_reserve[MAX_NR_ZONES];
    /**
     * 用于实现单一页框的特殊高速缓存。
     * 每内存管理区对每CPU都有一个。包含热高速缓存和冷高速缓存。
     * 内核使用这些列表来保存可用于满足实现的“新鲜”页。
     * 有些页帧很可能在CPU高速缓存中，因此可以快速访问，称之为热。
     * 未缓存的页帧称之为冷的。
     */
    struct per_cpu_pageset    pageset[NR_CPUS];

    /*
     * free areas of different sizes
     */
    /**
     * 保护该描述符的自旋锁
     */
    spinlock_t        lock;
    /**
     * 标识出管理区中的空闲页框块。
     * 包含11个元素，被伙伴系统使用。分别对应大小的1,2,4,8,16,32,128,256,512,1024连续空闲块的链表。
     * 第k个元素标识所有大小为2^k的空闲块。free_list字段指向双向循环链表的头。
     * free_list是free_area的内部结构，是个双向环回链表节点。
     */
    struct free_area    free_area[MAX_ORDER];

    /* 
     * 为了cache line对齐加的pad
     */
    ZONE_PADDING(_pad1_)

    /* Fields commonly accessed by the page reclaim scanner */
    /**
     * 活动以及非活动链表使用的自旋锁。
     */
    spinlock_t        lru_lock;    
    /**
     * 管理区中的活动页链表
     */
    struct list_head    active_list;
    /**
     * 管理区中的非活动页链表。
     */
    struct list_head    inactive_list;
    /**
     * 回收内存时需要扫描的活动页数。
     */
    unsigned long        nr_scan_active;
    /**
     * 回收内存时需要扫描的非活动页数目
     */
    unsigned long        nr_scan_inactive;
    /**
     * 管理区的活动链表上的页数目。
     */
    unsigned long        nr_active;
    /**
     * 管理区的非活动链表上的页数目。
     */
    unsigned long        nr_inactive;
    /**
     * 管理区内回收页框时使用的计数器。
     */
    unsigned long        pages_scanned;       /* since last reclaim */
    /**
     * 在管理区中填满不可回收页时此标志被置位
     */
    int            all_unreclaimable; /* All pages pinned */

    /*
     * prev_priority holds the scanning priority for this zone.  It is
     * defined as the scanning priority at which we achieved our reclaim
     * target at the previous try_to_free_pages() or balance_pgdat()
     * invokation.
     *
     * We use prev_priority as a measure of how much stress page reclaim is
     * under - it drives the swappiness decision: whether to unmap mapped
     * pages.
     *
     * temp_priority is used to remember the scanning priority at which
     * this zone was successfully refilled to free_pages == pages_high.
     *
     * Access to both these fields is quite racy even on uniprocessor.  But
     * it is expected to average out OK.
     */
    /**
     * 临时管理区的优先级。
     */
    int temp_priority;
    /**
     * 管理区优先级，范围在12和0之间。
     */
    int prev_priority;


    ZONE_PADDING(_pad2_)
    /* Rarely used or read-mostly fields */

    /*
     * wait_table        -- the array holding the hash table
     * wait_table_size    -- the size of the hash table array
     * wait_table_bits    -- wait_table_size == (1 << wait_table_bits)
     *
     * The purpose of all these is to keep track of the people
     * waiting for a page to become available and make them
     * runnable again when possible. The trouble is that this
     * consumes a lot of space, especially when so few things
     * wait on pages at a given time. So instead of using
     * per-page waitqueues, we use a waitqueue hash table.
     *
     * The bucket discipline is to sleep on the same queue when
     * colliding and wake all in that wait queue when removing.
     * When something wakes, it must check to be sure its page is
     * truly available, a la thundering herd. The cost of a
     * collision is great, but given the expected load of the
     * table, they should be so rare as to be outweighed by the
     * benefits from the saved space.
     *
     * __wait_on_page_locked() and unlock_page() in mm/filemap.c, are the
     * primary users of these fields, and in mm/page_alloc.c
     * free_area_init_core() performs the initialization of them.
     */
    /**
     * 进程等待队列的散列表。这些进程正在等待管理区中的某页。
     */
    wait_queue_head_t    * wait_table;
    /**
     * 等待队列散列表的大小。
     */
    unsigned long        wait_table_size;
    /**
     * 等待队列散列表数组的大小。值为2^order
     */
    unsigned long        wait_table_bits;

    /*
     * Discontig memory support fields.
     */
    /**
     * 内存节点。
     */
    struct pglist_data    *zone_pgdat;
    /** 
     * 指向管理区的第一个页描述符的指针。这个指针是数组mem_map的一个元素。
     */
    struct page        *zone_mem_map;
    /* zone_start_pfn == zone_start_paddr >> PAGE_SHIFT */
    /**
     * 管理区的第一个页框的下标。
     */
    unsigned long        zone_start_pfn;

    /**
     * 以页为单位的管理区的总大小，包含空洞。
     */
    unsigned long        spanned_pages;    /* total size, including holes */
    /**
     * 以页为单位的管理区的总大小，不包含空洞。
     */
    unsigned long        present_pages;    /* amount of memory (excluding holes) */

    /*
     * rarely used fields:
     */
    /**
     * 指针指向管理区的传统名称：DMA、NORMAL、HighMem
     */
    char            *name;
} ____cacheline_maxaligned_in_smp;

/*
 * Definitions unique to the original Linux SLAB allocator.
   slab描述符
 */

struct kmem_cache {
    struct array_cache __percpu *cpu_cache;/*本地cpu缓存池*/

/* 1) Cache tunables. Protected by slab_mutex */
    unsigned int batchcount;
    unsigned int limit;
    unsigned int shared;

    unsigned int size;
    struct reciprocal_value reciprocal_buffer_size;
/* 2) touched by every alloc & free from the backend */

    unsigned int flags;        /* constant flags */
    unsigned int num;        /* # of objs per slab：每个slab中object的数量 */

/* 3) cache_grow/shrink */
    /* order of pgs per slab (2^n) ：每个slab占用的页面数量*/
    unsigned int gfporder;

    /* force GFP flags, e.g. GFP_DMA */
    gfp_t allocflags;

    size_t colour;            /* cache colouring range：一个slab中不同cache line的数量 */
    unsigned int colour_off;    /* colour offset */
    struct kmem_cache *freelist_cache;/*打造单向链表*/
    unsigned int freelist_size;

    /* constructor func */
    void (*ctor)(void *obj);

/* 4) cache creation/removal */
    const char *name;/*slab描述符名字*/
    struct list_head list;
    int refcount;
    int object_size;/*onject对象的大小，每个kmem_cache可以个性化设置的*/
    int align;/*对齐长度*/

/* 5) statistics */
#ifdef CONFIG_DEBUG_SLAB
    unsigned long num_active;
    unsigned long num_allocations;
    unsigned long high_mark;
    unsigned long grown;
    unsigned long reaped;
    unsigned long errors;
    unsigned long max_freeable;
    unsigned long node_allocs;
    unsigned long node_frees;
    unsigned long node_overflow;
    atomic_t allochit;
    atomic_t allocmiss;
    atomic_t freehit;
    atomic_t freemiss;
#ifdef CONFIG_DEBUG_SLAB_LEAK
    atomic_t store_user_clean;
#endif

    /*
     * If debugging is enabled, then the allocator can add additional
     * fields and/or padding to every object. size contains the total
     * object size including these internal fields, the following two
     * variables contain the offset to the user object and its size.
     */
    int obj_offset;
#endif /* CONFIG_DEBUG_SLAB */

#ifdef CONFIG_MEMCG
    struct memcg_cache_params memcg_params;
#endif
#ifdef CONFIG_KASAN
    struct kasan_cache kasan_info;
#endif

#ifdef CONFIG_SLAB_FREELIST_RANDOM
    unsigned int *random_seq;
#endif
    /*slab链表*/
    struct kmem_cache_node *node[MAX_NUMNODES];
};

#ifndef CONFIG_SLOB
/*
 * The slab lists for all objects.
 */
struct kmem_cache_node {
    spinlock_t list_lock;

#ifdef CONFIG_SLAB
    struct list_head slabs_partial;    /* partial list first, better asm code */
    struct list_head slabs_full;
    struct list_head slabs_free;
    unsigned long num_slabs;
    unsigned long free_objects;
    unsigned int free_limit;
    unsigned int colour_next;    /* Per-node cache coloring */
    struct array_cache *shared;    /* shared per node */
    struct alien_cache **alien;    /* on other nodes */
    unsigned long next_reap;    /* updated without locking */
    int free_touched;        /* updated without locking */
#endif

#ifdef CONFIG_SLUB
    unsigned long nr_partial;
    struct list_head partial;
#ifdef CONFIG_SLUB_DEBUG
    atomic_long_t nr_slabs;
    atomic_long_t total_objects;
    struct list_head full;
#endif
#endif

};