http {

    include       mime.types;

    default_type  application/octet-stream;

    #log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '

    #                  '$status $body_bytes_sent "$http_referer" '

    #                  '"$http_user_agent" "$http_x_forwarded_for"';

    #access_log  logs/access.log  main;

    sendfile        on;

    #tcp_nopush     on;

    #keepalive_timeout  0;

    keepalive_timeout  65;

    #gzip  on;

    server {

        listen       8085;

        server_name  localhost;

        #charset koi8-r;

        #access_log  logs/host.access.log  main;

        location /tohello {

            # 将请求转发给另一个服务器

            proxy_pass http://localhost:8081/hello;

            # 保持请求端信息

            proxy_set_header Host $host;

            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;

        }

        location / {

            root   html;

            index  index.html index.htm;

        }

    }

    # 后续可以添加无数个server 扩展

}

// http/modules/ngx_http_proxy_module.c

ngx_module_t  ngx_http_proxy_module;

// 所有支持的操作命令定义,想查看完整定义，请点击后续代码块

static ngx_command_t  ngx_http_proxy_commands[] = {

    { ngx_string("proxy_pass"),

      NGX_HTTP_LOC_CONF|NGX_HTTP_LIF_CONF|NGX_HTTP_LMT_CONF|NGX_CONF_TAKE1,

      ngx_http_proxy_pass,

      NGX_HTTP_LOC_CONF_OFFSET,

      0,

      NULL },

    { ngx_string("proxy_set_header"),

      NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE2,

      ngx_conf_set_keyval_slot,

      NGX_HTTP_LOC_CONF_OFFSET,

      offsetof(ngx_http_proxy_loc_conf_t, headers_source),

      NULL },

        ...

      ngx_null_command

};

static ngx_http_module_t  ngx_http_proxy_module_ctx = {

    ngx_http_proxy_add_variables,          /* preconfiguration */

    NULL,                                  /* postconfiguration */

    ngx_http_proxy_create_main_conf,       /* create main configuration */

    NULL,                                  /* init main configuration */

    NULL,                                  /* create server configuration */

    NULL,                                  /* merge server configuration */

    ngx_http_proxy_create_loc_conf,        /* create location configuration */

    ngx_http_proxy_merge_loc_conf          /* merge location configuration */

};

// 暴露模块服务

ngx_module_t  ngx_http_proxy_module = {

    NGX_MODULE_V1,

    &ngx_http_proxy_module_ctx,            /* module context */

    ngx_http_proxy_commands,               /* module directives */

    NGX_HTTP_MODULE,                       /* module type */

    NULL,                                  /* init master */

    NULL,                                  /* init module */

    NULL,                                  /* init process */

    NULL,                                  /* init thread */

    NULL,                                  /* exit thread */

    NULL,                                  /* exit process */

    NULL,                                  /* exit master */

    NGX_MODULE_V1_PADDING

};

// http/ngx_http_core_module.c

ngx_int_t

ngx_http_core_content_phase(ngx_http_request_t *r,

    ngx_http_phase_handler_t *ph)

{

    size_t     root;

    ngx_int_t  rc;

    ngx_str_t  path;

    if (r->content_handler) {

        r->write_event_handler = ngx_http_request_empty_handler;

        // 调用 content_handler 进行转发处理

        ngx_http_finalize_request(r, r->content_handler(r));

        return NGX_OK;

    }

    ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,

                   "content phase: %ui", r->phase_handler);

    rc = ph->handler(r);

    if (rc != NGX_DECLINED) {

        ngx_http_finalize_request(r, rc);

        return NGX_OK;

    }

    /* rc == NGX_DECLINED */

    ph++;

    if (ph->checker) {

        r->phase_handler++;

        return NGX_AGAIN;

    }

    /* no content handler was found */

    if (r->uri.data[r->uri.len - 1] == '/') {

        if (ngx_http_map_uri_to_path(r, &path, &root, 0) != NULL) {

            ngx_log_error(NGX_LOG_ERR, r->connection->log, 0,

                          "directory index of \"%s\" is forbidden", path.data);

        }

        ngx_http_finalize_request(r, NGX_HTTP_FORBIDDEN);

        return NGX_OK;

    }

    ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, "no handler found");

    ngx_http_finalize_request(r, NGX_HTTP_NOT_FOUND);

    return NGX_OK;

}

// http/modules/ngx_http_proxy_module.c

// 代理功能入口

static ngx_int_t

ngx_http_proxy_handler(ngx_http_request_t *r)

{

    ngx_int_t                    rc;

    ngx_http_upstream_t         *u;

    ngx_http_proxy_ctx_t        *ctx;

    ngx_http_proxy_loc_conf_t   *plcf;

#if (NGX_HTTP_CACHE)

    ngx_http_proxy_main_conf_t  *pmcf;

#endif

    // 创建upstream, 即转发流准备

    if (ngx_http_upstream_create(r) != NGX_OK) {

        return NGX_HTTP_INTERNAL_SERVER_ERROR;

    }

    ctx = ngx_pcalloc(r->pool, sizeof(ngx_http_proxy_ctx_t));

    if (ctx == NULL) {

        return NGX_HTTP_INTERNAL_SERVER_ERROR;

    }

    // 将创建好的上下文信息赋给 r->ctx 中

    // r->ctx[ngx_http_proxy_module.ctx_index] = ctx;

    ngx_http_set_ctx(r, ctx, ngx_http_proxy_module);

    plcf = ngx_http_get_module_loc_conf(r, ngx_http_proxy_module);

    // 设置upstream 信息

    u = r->upstream;

    if (plcf->proxy_lengths == NULL) {

        // {key_start = {len = 21, data = 0x8000b4110 "http://localhost:8081/hello"},

        // schema = {len = 7, data = 0x8000b4110 "http://localhost:8081/hello"},

        // host_header = {len = 14, data = 0x8000b4117 "localhost:8081/hello"},

        // port = {len = 4, data = 0x8000b4121 "8081/hello"},

        // uri = {len = 6, data = 0x8000b4125 "/hello"}}

        ctx->vars = plcf->vars;

        u->schema = plcf->vars.schema;

#if (NGX_HTTP_SSL)

        u->ssl = (plcf->upstream.ssl != NULL);

#endif

    } else {

        if (ngx_http_proxy_eval(r, ctx, plcf) != NGX_OK) {

            return NGX_HTTP_INTERNAL_SERVER_ERROR;

        }

    }

    u->output.tag = (ngx_buf_tag_t) &ngx_http_proxy_module;

    u->conf = &plcf->upstream;

#if (NGX_HTTP_CACHE)

    pmcf = ngx_http_get_module_main_conf(r, ngx_http_proxy_module);

    u->caches = &pmcf->caches;

    u->create_key = ngx_http_proxy_create_key;

#endif

    u->create_request = ngx_http_proxy_create_request;

    u->reinit_request = ngx_http_proxy_reinit_request;

    u->process_header = ngx_http_proxy_process_status_line;

    u->abort_request = ngx_http_proxy_abort_request;

    u->finalize_request = ngx_http_proxy_finalize_request;

    r->state = 0;

    // 重定向设置

    if (plcf->redirects) {

        u->rewrite_redirect = ngx_http_proxy_rewrite_redirect;

    }

    if (plcf->cookie_domains || plcf->cookie_paths) {

        u->rewrite_cookie = ngx_http_proxy_rewrite_cookie;

    }

    u->buffering = plcf->upstream.buffering;

    u->pipe = ngx_pcalloc(r->pool, sizeof(ngx_event_pipe_t));

    if (u->pipe == NULL) {

        return NGX_HTTP_INTERNAL_SERVER_ERROR;

    }

    u->pipe->input_filter = ngx_http_proxy_copy_filter;

    u->pipe->input_ctx = r;

    u->input_filter_init = ngx_http_proxy_input_filter_init;

    u->input_filter = ngx_http_proxy_non_buffered_copy_filter;

    u->input_filter_ctx = r;

    u->accel = 1;

    if (!plcf->upstream.request_buffering

        && plcf->body_values == NULL && plcf->upstream.pass_request_body

        && (!r->headers_in.chunked

            || plcf->http_version == NGX_HTTP_VERSION_11))

    {

        r->request_body_no_buffering = 1;

    }

    // 重要: 读取客户端请求数据

    // ngx_http_upstream_init 被作为处理器传入

    rc = ngx_http_read_client_request_body(r, ngx_http_upstream_init);

    if (rc >= NGX_HTTP_SPECIAL_RESPONSE) {

        return rc;

    }

    return NGX_DONE;

}

// http/ngx_http_upstream.c

ngx_int_t

ngx_http_upstream_create(ngx_http_request_t *r)

{

    ngx_http_upstream_t  *u;

    u = r->upstream;

    if (u && u->cleanup) {

        r->main->count++;

        ngx_http_upstream_cleanup(r);

    }

    u = ngx_pcalloc(r->pool, sizeof(ngx_http_upstream_t));

    if (u == NULL) {

        return NGX_ERROR;

    }

    r->upstream = u;

    u->peer.log = r->connection->log;

    u->peer.log_error = NGX_ERROR_ERR;

#if (NGX_HTTP_CACHE)

    r->cache = NULL;

#endif

    // header 被设置为-1, 后续作区分处理

    u->headers_in.content_length_n = -1;

    u->headers_in.last_modified_time = -1;

    return NGX_OK;

}

// http/ngx_http_request_body.c

// 通用读取请求并处理流程

ngx_int_t

ngx_http_read_client_request_body(ngx_http_request_t *r,

    ngx_http_client_body_handler_pt post_handler)

{

    size_t                     preread;

    ssize_t                    size;

    ngx_int_t                  rc;

    ngx_buf_t                 *b;

    ngx_chain_t                out;

    ngx_http_request_body_t   *rb;

    ngx_http_core_loc_conf_t  *clcf;

    r->main->count++;

    if (r != r->main || r->request_body || r->discard_body) {

        r->request_body_no_buffering = 0;

        post_handler(r);

        return NGX_OK;

    }

    if (ngx_http_test_expect(r) != NGX_OK) {

        rc = NGX_HTTP_INTERNAL_SERVER_ERROR;

        goto done;

    }

    rb = ngx_pcalloc(r->pool, sizeof(ngx_http_request_body_t));

    if (rb == NULL) {

        rc = NGX_HTTP_INTERNAL_SERVER_ERROR;

        goto done;

    }

    /*

     * set by ngx_pcalloc():

     *

     *     rb->bufs = NULL;

     *     rb->buf = NULL;

     *     rb->free = NULL;

     *     rb->busy = NULL;

     *     rb->chunked = NULL;

     */

    rb->rest = -1;

    rb->post_handler = post_handler;

    r->request_body = rb;

    if (r->headers_in.content_length_n < 0 && !r->headers_in.chunked) {

        r->request_body_no_buffering = 0;

        // header为-1, proxy 会直接走此处, 即转身 upstream 处理

        post_handler(r);

        return NGX_OK;

    }

    ...

}

// http/ngx_http_upstream.c

void

ngx_http_upstream_init(ngx_http_request_t *r)

{

    ngx_connection_t     *c;

    c = r->connection;

    ngx_log_debug1(NGX_LOG_DEBUG_HTTP, c->log, 0,

                   "http init upstream, client timer: %d", c->read->timer_set);

#if (NGX_HTTP_V2)

    if (r->stream) {

        ngx_http_upstream_init_request(r);

        return;

    }

#endif

    if (c->read->timer_set) {

        ngx_del_timer(c->read);

    }

    if (ngx_event_flags & NGX_USE_CLEAR_EVENT) {

        if (!c->write->active) {

            if (ngx_add_event(c->write, NGX_WRITE_EVENT, NGX_CLEAR_EVENT)

                == NGX_ERROR)

            {

                ngx_http_finalize_request(r, NGX_HTTP_INTERNAL_SERVER_ERROR);

                return;

            }

        }

    }

    ngx_http_upstream_init_request(r);

}

// http/ngx_http_upstream.c

static void

ngx_http_upstream_init_request(ngx_http_request_t *r)

{

    ngx_str_t                      *host;

    ngx_uint_t                      i;

    ngx_resolver_ctx_t             *ctx, temp;

    ngx_http_cleanup_t             *cln;

    ngx_http_upstream_t            *u;

    ngx_http_core_loc_conf_t       *clcf;

    ngx_http_upstream_srv_conf_t   *uscf, **uscfp;

    ngx_http_upstream_main_conf_t  *umcf;

    if (r->aio) {

        return;

    }

    u = r->upstream;

#if (NGX_HTTP_CACHE)

    if (u->conf->cache) {

        ngx_int_t  rc;

        rc = ngx_http_upstream_cache(r, u);

        if (rc == NGX_BUSY) {

            r->write_event_handler = ngx_http_upstream_init_request;

            return;

        }

        r->write_event_handler = ngx_http_request_empty_handler;

        if (rc == NGX_ERROR) {

            ngx_http_finalize_request(r, NGX_HTTP_INTERNAL_SERVER_ERROR);

            return;

        }

        if (rc == NGX_OK) {

            rc = ngx_http_upstream_cache_send(r, u);

            if (rc == NGX_DONE) {

                return;

            }

            if (rc == NGX_HTTP_UPSTREAM_INVALID_HEADER) {

                rc = NGX_DECLINED;

                r->cached = 0;

                u->buffer.start = NULL;

                u->cache_status = NGX_HTTP_CACHE_MISS;

                u->request_sent = 1;

            }

        }

        if (rc != NGX_DECLINED) {

            ngx_http_finalize_request(r, rc);

            return;

        }

    }

#endif

    u->store = u->conf->store;

    if (!u->store && !r->post_action && !u->conf->ignore_client_abort) {

        r->read_event_handler = ngx_http_upstream_rd_check_broken_connection;

        r->write_event_handler = ngx_http_upstream_wr_check_broken_connection;

    }

    if (r->request_body) {

        u->request_bufs = r->request_body->bufs;

    }

    // 创建代理请求, 此处为 ngx_http_proxy_create_request

    // {len = 3, data = 0x8000a9700 "GET /tohello/getUsers?pageNum=1&pageSize=2 HTTP/1.1\r\nHost"}

    if (u->create_request(r) != NGX_OK) {

        ngx_http_finalize_request(r, NGX_HTTP_INTERNAL_SERVER_ERROR);

        return;

    }

    if (ngx_http_upstream_set_local(r, u, u->conf->local) != NGX_OK) {

        ngx_http_finalize_request(r, NGX_HTTP_INTERNAL_SERVER_ERROR);

        return;

    }

    if (u->conf->socket_keepalive) {

        u->peer.so_keepalive = 1;

    }

    clcf = ngx_http_get_module_loc_conf(r, ngx_http_core_module);

    u->output.alignment = clcf->directio_alignment;

    u->output.pool = r->pool;

    u->output.bufs.num = 1;

    u->output.bufs.size = clcf->client_body_buffer_size;

    if (u->output.output_filter == NULL) {

        u->output.output_filter = ngx_chain_writer;

        u->output.filter_ctx = &u->writer;

    }

    u->writer.pool = r->pool;

    if (r->upstream_states == NULL) {

        r->upstream_states = ngx_array_create(r->pool, 1,

                                            sizeof(ngx_http_upstream_state_t));

        if (r->upstream_states == NULL) {

            ngx_http_finalize_request(r, NGX_HTTP_INTERNAL_SERVER_ERROR);

            return;

        }

    } else {

        u->state = ngx_array_push(r->upstream_states);

        if (u->state == NULL) {

            ngx_http_upstream_finalize_request(r, u,

                                               NGX_HTTP_INTERNAL_SERVER_ERROR);

            return;

        }

        ngx_memzero(u->state, sizeof(ngx_http_upstream_state_t));

    }

    // 清理数据

    cln = ngx_http_cleanup_add(r, 0);

    if (cln == NULL) {

        ngx_http_finalize_request(r, NGX_HTTP_INTERNAL_SERVER_ERROR);

        return;

    }

    cln->handler = ngx_http_upstream_cleanup;

    cln->data = r;

    u->cleanup = &cln->handler;

    if (u->resolved == NULL) {

        uscf = u->conf->upstream;

    } else {

#if (NGX_HTTP_SSL)

        u->ssl_name = u->resolved->host;

#endif

        host = &u->resolved->host;

        umcf = ngx_http_get_module_main_conf(r, ngx_http_upstream_module);

        uscfp = umcf->upstreams.elts;

        for (i = 0; i < umcf->upstreams.nelts; i++) {

            uscf = uscfp[i];

            if (uscf->host.len == host->len

                && ((uscf->port == 0 && u->resolved->no_port)

                     || uscf->port == u->resolved->port)

                && ngx_strncasecmp(uscf->host.data, host->data, host->len) == 0)

            {

                goto found;

            }

        }

        if (u->resolved->sockaddr) {

            if (u->resolved->port == 0

                && u->resolved->sockaddr->sa_family != AF_UNIX)

            {

                ngx_log_error(NGX_LOG_ERR, r->connection->log, 0,

                              "no port in upstream \"%V\"", host);

                ngx_http_upstream_finalize_request(r, u,

                                               NGX_HTTP_INTERNAL_SERVER_ERROR);

                return;

            }

            if (ngx_http_upstream_create_round_robin_peer(r, u->resolved)

                != NGX_OK)

            {

                ngx_http_upstream_finalize_request(r, u,

                                               NGX_HTTP_INTERNAL_SERVER_ERROR);

                return;

            }

            ngx_http_upstream_connect(r, u);

            return;

        }

        if (u->resolved->port == 0) {

            ngx_log_error(NGX_LOG_ERR, r->connection->log, 0,

                          "no port in upstream \"%V\"", host);

            ngx_http_upstream_finalize_request(r, u,

                                               NGX_HTTP_INTERNAL_SERVER_ERROR);

            return;

        }

        temp.name = *host;

        ctx = ngx_resolve_start(clcf->resolver, &temp);

        if (ctx == NULL) {

            ngx_http_upstream_finalize_request(r, u,

                                               NGX_HTTP_INTERNAL_SERVER_ERROR);

            return;

        }

        if (ctx == NGX_NO_RESOLVER) {

            ngx_log_error(NGX_LOG_ERR, r->connection->log, 0,

                          "no resolver defined to resolve %V", host);

            ngx_http_upstream_finalize_request(r, u, NGX_HTTP_BAD_GATEWAY);

            return;

        }

        ctx->name = *host;

        ctx->handler = ngx_http_upstream_resolve_handler;

        ctx->data = r;

        ctx->timeout = clcf->resolver_timeout;

        u->resolved->ctx = ctx;

        if (ngx_resolve_name(ctx) != NGX_OK) {

            u->resolved->ctx = NULL;

            ngx_http_upstream_finalize_request(r, u,

                                               NGX_HTTP_INTERNAL_SERVER_ERROR);

            return;

        }

        return;

    }

found:

    if (uscf == NULL) {

        ngx_log_error(NGX_LOG_ALERT, r->connection->log, 0,

                      "no upstream configuration");

        ngx_http_upstream_finalize_request(r, u,

                                           NGX_HTTP_INTERNAL_SERVER_ERROR);

        return;

    }

    u->upstream = uscf;

#if (NGX_HTTP_SSL)

    u->ssl_name = uscf->host;

#endif

    // 初始化连接点数据

    // 默认为: ngx_http_upstream_init_round_robin_peer

    if (uscf->peer.init(r, uscf) != NGX_OK) {

        ngx_http_upstream_finalize_request(r, u,

                                           NGX_HTTP_INTERNAL_SERVER_ERROR);

        return;

    }

    u->peer.start_time = ngx_current_msec;

    if (u->conf->next_upstream_tries

        && u->peer.tries > u->conf->next_upstream_tries)

    {

        u->peer.tries = u->conf->next_upstream_tries;

    }

    // 连接到upstream 中, 默认是 round-robin

    ngx_http_upstream_connect(r, u);

}

// http/ngx_http_upstream.c

static void

ngx_http_upstream_connect(ngx_http_request_t *r, ngx_http_upstream_t *u)

{

    ngx_int_t          rc;

    ngx_connection_t  *c;

    r->connection->log->action = "connecting to upstream";

    if (u->state && u->state->response_time == (ngx_msec_t) -1) {

        u->state->response_time = ngx_current_msec - u->start_time;

    }

    u->state = ngx_array_push(r->upstream_states);

    if (u->state == NULL) {

        ngx_http_upstream_finalize_request(r, u,

                                           NGX_HTTP_INTERNAL_SERVER_ERROR);

        return;

    }

    ngx_memzero(u->state, sizeof(ngx_http_upstream_state_t));

    u->start_time = ngx_current_msec;

    u->state->response_time = (ngx_msec_t) -1;

    u->state->connect_time = (ngx_msec_t) -1;

    u->state->header_time = (ngx_msec_t) -1;

    // 连接无端socket, count++

    rc = ngx_event_connect_peer(&u->peer);

    ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,

                   "http upstream connect: %i", rc);

    if (rc == NGX_ERROR) {

        ngx_http_upstream_finalize_request(r, u,

                                           NGX_HTTP_INTERNAL_SERVER_ERROR);

        return;

    }

    u->state->peer = u->peer.name;

    if (rc == NGX_BUSY) {

        ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, "no live upstreams");

        ngx_http_upstream_next(r, u, NGX_HTTP_UPSTREAM_FT_NOLIVE);

        return;

    }

    if (rc == NGX_DECLINED) {

        ngx_http_upstream_next(r, u, NGX_HTTP_UPSTREAM_FT_ERROR);

        return;

    }

    /* rc == NGX_OK || rc == NGX_AGAIN || rc == NGX_DONE */

    c = u->peer.connection;

    c->requests++;

    c->data = r;

    // 读写事件处理器设置为 ngx_http_upstream_handler

    c->write->handler = ngx_http_upstream_handler;

    c->read->handler = ngx_http_upstream_handler;

    u->write_event_handler = ngx_http_upstream_send_request_handler;

    u->read_event_handler = ngx_http_upstream_process_header;

    c->sendfile &= r->connection->sendfile;

    u->output.sendfile = c->sendfile;

    if (r->connection->tcp_nopush == NGX_TCP_NOPUSH_DISABLED) {

        c->tcp_nopush = NGX_TCP_NOPUSH_DISABLED;

    }

    if (c->pool == NULL) {

        /* we need separate pool here to be able to cache SSL connections */

        c->pool = ngx_create_pool(128, r->connection->log);

        if (c->pool == NULL) {

            ngx_http_upstream_finalize_request(r, u,

                                               NGX_HTTP_INTERNAL_SERVER_ERROR);

            return;

        }

    }

    c->log = r->connection->log;

    c->pool->log = c->log;

    c->read->log = c->log;

    c->write->log = c->log;

    /* init or reinit the ngx_output_chain() and ngx_chain_writer() contexts */

    u->writer.out = NULL;

    u->writer.last = &u->writer.out;

    u->writer.connection = c;

    u->writer.limit = 0;

    if (u->request_sent) {

        if (ngx_http_upstream_reinit(r, u) != NGX_OK) {

            ngx_http_upstream_finalize_request(r, u,

                                               NGX_HTTP_INTERNAL_SERVER_ERROR);

            return;

        }

    }

    if (r->request_body

        && r->request_body->buf

        && r->request_body->temp_file

        && r == r->main)

    {

        /*

         * the r->request_body->buf can be reused for one request only,

         * the subrequests should allocate their own temporary bufs

         */

        u->output.free = ngx_alloc_chain_link(r->pool);

        if (u->output.free == NULL) {

            ngx_http_upstream_finalize_request(r, u,

                                               NGX_HTTP_INTERNAL_SERVER_ERROR);

            return;

        }

        u->output.free->buf = r->request_body->buf;

        u->output.free->next = NULL;

        u->output.allocated = 1;

        r->request_body->buf->pos = r->request_body->buf->start;

        r->request_body->buf->last = r->request_body->buf->start;

        r->request_body->buf->tag = u->output.tag;

    }

    u->request_sent = 0;

    u->request_body_sent = 0;

    u->request_body_blocked = 0;

    // 未处理完成，等待下一次事件通知

    if (rc == NGX_AGAIN) {

        ngx_add_timer(c->write, u->conf->connect_timeout);

        return;

    }

#if (NGX_HTTP_SSL)

    if (u->ssl && c->ssl == NULL) {

        ngx_http_upstream_ssl_init_connection(r, u, c);

        return;

    }

#endif

    // 发送请求到目标端

    ngx_http_upstream_send_request(r, u, 1);

}

// http/ngx_http_upstream.c

static void

ngx_http_upstream_send_request(ngx_http_request_t *r, ngx_http_upstream_t *u,

    ngx_uint_t do_write)

{

    ngx_int_t          rc;

    ngx_connection_t  *c;

    c = u->peer.connection;

    ngx_log_debug0(NGX_LOG_DEBUG_HTTP, c->log, 0,

                   "http upstream send request");

    if (u->state->connect_time == (ngx_msec_t) -1) {

        u->state->connect_time = ngx_current_msec - u->start_time;

    }

    // 测试发送数据ok

    if (!u->request_sent && ngx_http_upstream_test_connect(c) != NGX_OK) {

        ngx_http_upstream_next(r, u, NGX_HTTP_UPSTREAM_FT_ERROR);

        return;

    }

    c->log->action = "sending request to upstream";

    // 发送数据

    rc = ngx_http_upstream_send_request_body(r, u, do_write);

    if (rc == NGX_ERROR) {

        ngx_http_upstream_next(r, u, NGX_HTTP_UPSTREAM_FT_ERROR);

        return;

    }

    if (rc >= NGX_HTTP_SPECIAL_RESPONSE) {

        ngx_http_upstream_finalize_request(r, u, rc);

        return;

    }

    if (rc == NGX_AGAIN) {

        if (!c->write->ready || u->request_body_blocked) {

            ngx_add_timer(c->write, u->conf->send_timeout);

        } else if (c->write->timer_set) {

            ngx_del_timer(c->write);

        }

        if (ngx_handle_write_event(c->write, u->conf->send_lowat) != NGX_OK) {

            ngx_http_upstream_finalize_request(r, u,

                                               NGX_HTTP_INTERNAL_SERVER_ERROR);

            return;

        }

        if (c->write->ready && c->tcp_nopush == NGX_TCP_NOPUSH_SET) {

            if (ngx_tcp_push(c->fd) == -1) {

                ngx_log_error(NGX_LOG_CRIT, c->log, ngx_socket_errno,

                              ngx_tcp_push_n " failed");

                ngx_http_upstream_finalize_request(r, u,

                                               NGX_HTTP_INTERNAL_SERVER_ERROR);

                return;

            }

            c->tcp_nopush = NGX_TCP_NOPUSH_UNSET;

        }

        return;

    }

    /* rc == NGX_OK */

    if (c->write->timer_set) {

        ngx_del_timer(c->write);

    }

    if (c->tcp_nopush == NGX_TCP_NOPUSH_SET) {

        if (ngx_tcp_push(c->fd) == -1) {

            ngx_log_error(NGX_LOG_CRIT, c->log, ngx_socket_errno,

                          ngx_tcp_push_n " failed");

            ngx_http_upstream_finalize_request(r, u,

                                               NGX_HTTP_INTERNAL_SERVER_ERROR);

            return;

        }

        c->tcp_nopush = NGX_TCP_NOPUSH_UNSET;

    }

    if (!u->conf->preserve_output) {

        u->write_event_handler = ngx_http_upstream_dummy_handler;

    }

    // 写事件

    if (ngx_handle_write_event(c->write, 0) != NGX_OK) {

        ngx_http_upstream_finalize_request(r, u,

                                           NGX_HTTP_INTERNAL_SERVER_ERROR);

        return;

    }

    if (!u->request_body_sent) {

        u->request_body_sent = 1;

        if (u->header_sent) {

            return;

        }

        // 注册读超时定时器，避免等等超时

        ngx_add_timer(c->read, u->conf->read_timeout);

        // 如果已响应，立即处理，否则异步等待后续事件通知

        if (c->read->ready) {

            ngx_http_upstream_process_header(r, u);

            return;

        }

    }

}

// http/ngx_http_upstream.c

static void

ngx_http_upstream_handler(ngx_event_t *ev)

{

    ngx_connection_t     *c;

    ngx_http_request_t   *r;

    ngx_http_upstream_t  *u;

    c = ev->data;

    r = c->data;

    u = r->upstream;

    c = r->connection;

    ngx_http_set_log_request(c->log, r);

    ngx_log_debug2(NGX_LOG_DEBUG_HTTP, c->log, 0,

                   "http upstream request: \"%V?%V\"", &r->uri, &r->args);

    if (ev->delayed && ev->timedout) {

        ev->delayed = 0;

        ev->timedout = 0;

    }

    // 写就绪、读就绪

    if (ev->write) {

        u->write_event_handler(r, u);

    } else {

        u->read_event_handler(r, u);

    }

    ngx_http_run_posted_requests(c);

}

// 接收目标服务器返回值并处理

// http/ngx_http_upstream.c

static void

ngx_http_upstream_process_header(ngx_http_request_t *r, ngx_http_upstream_t *u)

{

    ssize_t            n;

    ngx_int_t          rc;

    ngx_connection_t  *c;

    c = u->peer.connection;

    ngx_log_debug0(NGX_LOG_DEBUG_HTTP, c->log, 0,

                   "http upstream process header");

    c->log->action = "reading response header from upstream";

    // 超时处理

    if (c->read->timedout) {

        ngx_http_upstream_next(r, u, NGX_HTTP_UPSTREAM_FT_TIMEOUT);

        return;

    }

    if (!u->request_sent && ngx_http_upstream_test_connect(c) != NGX_OK) {

        ngx_http_upstream_next(r, u, NGX_HTTP_UPSTREAM_FT_ERROR);

        return;

    }

    if (u->buffer.start == NULL) {

        u->buffer.start = ngx_palloc(r->pool, u->conf->buffer_size);

        if (u->buffer.start == NULL) {

            ngx_http_upstream_finalize_request(r, u,

                                               NGX_HTTP_INTERNAL_SERVER_ERROR);

            return;

        }

        u->buffer.pos = u->buffer.start;

        u->buffer.last = u->buffer.start;

        u->buffer.end = u->buffer.start + u->conf->buffer_size;

        u->buffer.temporary = 1;

        u->buffer.tag = u->output.tag;

        if (ngx_list_init(&u->headers_in.headers, r->pool, 8,

                          sizeof(ngx_table_elt_t))

            != NGX_OK)

        {

            ngx_http_upstream_finalize_request(r, u,

                                               NGX_HTTP_INTERNAL_SERVER_ERROR);

            return;

        }

        if (ngx_list_init(&u->headers_in.trailers, r->pool, 2,

                          sizeof(ngx_table_elt_t))

            != NGX_OK)

        {

            ngx_http_upstream_finalize_request(r, u,

                                               NGX_HTTP_INTERNAL_SERVER_ERROR);

            return;

        }

#if (NGX_HTTP_CACHE)

        if (r->cache) {

            u->buffer.pos += r->cache->header_start;

            u->buffer.last = u->buffer.pos;

        }

#endif

    }

    for ( ;; ) {

        // 循环读取目标服务器传回的数据

        n = c->recv(c, u->buffer.last, u->buffer.end - u->buffer.last);

        if (n == NGX_AGAIN) {

#if 0

            ngx_add_timer(rev, u->read_timeout);

#endif

            if (ngx_handle_read_event(c->read, 0) != NGX_OK) {

                ngx_http_upstream_finalize_request(r, u,

                                               NGX_HTTP_INTERNAL_SERVER_ERROR);

                return;

            }

            return;

        }

        if (n == 0) {

            ngx_log_error(NGX_LOG_ERR, c->log, 0,

                          "upstream prematurely closed connection");

        }

        if (n == NGX_ERROR || n == 0) {

            ngx_http_upstream_next(r, u, NGX_HTTP_UPSTREAM_FT_ERROR);

            return;

        }

        u->state->bytes_received += n;

        u->buffer.last += n;

#if 0

        u->valid_header_in = 0;

        u->peer.cached = 0;

#endif

        // 处理header信息

        // 该prcocess_header由前面做好的设置, ngx_http_proxy_process_status_line

        rc = u->process_header(r);

        if (rc == NGX_AGAIN) {

            if (u->buffer.last == u->buffer.end) {

                ngx_log_error(NGX_LOG_ERR, c->log, 0,

                              "upstream sent too big header");

                ngx_http_upstream_next(r, u,

                                       NGX_HTTP_UPSTREAM_FT_INVALID_HEADER);

                return;

            }

            continue;

        }

        break;

    }

    if (rc == NGX_HTTP_UPSTREAM_INVALID_HEADER) {

        ngx_http_upstream_next(r, u, NGX_HTTP_UPSTREAM_FT_INVALID_HEADER);

        return;

    }

    if (rc == NGX_ERROR) {

        ngx_http_upstream_finalize_request(r, u,

                                           NGX_HTTP_INTERNAL_SERVER_ERROR);

        return;

    }

    /* rc == NGX_OK */

    u->state->header_time = ngx_current_msec - u->start_time;

    if (u->headers_in.status_n >= NGX_HTTP_SPECIAL_RESPONSE) {

        if (ngx_http_upstream_test_next(r, u) == NGX_OK) {

            return;

        }

        if (ngx_http_upstream_intercept_errors(r, u) == NGX_OK) {

            return;

        }

    }

    // 处理header信息

    if (ngx_http_upstream_process_headers(r, u) != NGX_OK) {

        return;

    }

    // 响应客户端

    ngx_http_upstream_send_response(r, u);

}

// http/ngx_http_upstream.c

// 发送数据到客户端

static void

ngx_http_upstream_send_response(ngx_http_request_t *r, ngx_http_upstream_t *u)

{

    ssize_t                    n;

    ngx_int_t                  rc;

    ngx_event_pipe_t          *p;

    ngx_connection_t          *c;

    ngx_http_core_loc_conf_t  *clcf;

    // 发送header

    rc = ngx_http_send_header(r);

    if (rc == NGX_ERROR || rc > NGX_OK || r->post_action) {

        ngx_http_upstream_finalize_request(r, u, rc);

        return;

    }

    u->header_sent = 1;

    if (u->upgrade) {

#if (NGX_HTTP_CACHE)

        if (r->cache) {

            ngx_http_file_cache_free(r->cache, u->pipe->temp_file);

        }

#endif

        ngx_http_upstream_upgrade(r, u);

        return;

    }

    c = r->connection;

    if (r->header_only) {

        if (!u->buffering) {

            ngx_http_upstream_finalize_request(r, u, rc);

            return;

        }

        if (!u->cacheable && !u->store) {

            ngx_http_upstream_finalize_request(r, u, rc);

            return;

        }

        u->pipe->downstream_error = 1;

    }

    if (r->request_body && r->request_body->temp_file

        && r == r->main && !r->preserve_body

        && !u->conf->preserve_output)

    {

        ngx_pool_run_cleanup_file(r->pool, r->request_body->temp_file->file.fd);

        r->request_body->temp_file->file.fd = NGX_INVALID_FILE;

    }

    clcf = ngx_http_get_module_loc_conf(r, ngx_http_core_module);

    if (!u->buffering) {

#if (NGX_HTTP_CACHE)

        if (r->cache) {

            ngx_http_file_cache_free(r->cache, u->pipe->temp_file);

        }

#endif

        if (u->input_filter == NULL) {

            u->input_filter_init = ngx_http_upstream_non_buffered_filter_init;

            u->input_filter = ngx_http_upstream_non_buffered_filter;

            u->input_filter_ctx = r;

        }

        u->read_event_handler = ngx_http_upstream_process_non_buffered_upstream;

        r->write_event_handler =

                             ngx_http_upstream_process_non_buffered_downstream;

        r->limit_rate = 0;

        r->limit_rate_set = 1;

        if (u->input_filter_init(u->input_filter_ctx) == NGX_ERROR) {

            ngx_http_upstream_finalize_request(r, u, NGX_ERROR);

            return;

        }

        if (clcf->tcp_nodelay && ngx_tcp_nodelay(c) != NGX_OK) {

            ngx_http_upstream_finalize_request(r, u, NGX_ERROR);

            return;

        }

        n = u->buffer.last - u->buffer.pos;

        if (n) {

            u->buffer.last = u->buffer.pos;

            u->state->response_length += n;

            if (u->input_filter(u->input_filter_ctx, n) == NGX_ERROR) {

                ngx_http_upstream_finalize_request(r, u, NGX_ERROR);

                return;

            }

            ngx_http_upstream_process_non_buffered_downstream(r);

        } else {

            u->buffer.pos = u->buffer.start;

            u->buffer.last = u->buffer.start;

            if (ngx_http_send_special(r, NGX_HTTP_FLUSH) == NGX_ERROR) {

                ngx_http_upstream_finalize_request(r, u, NGX_ERROR);

                return;

            }

            if (u->peer.connection->read->ready || u->length == 0) {

                ngx_http_upstream_process_non_buffered_upstream(r, u);

            }

        }

        return;

    }

    /* TODO: preallocate event_pipe bufs, look "Content-Length" */

#if (NGX_HTTP_CACHE)

    if (r->cache && r->cache->file.fd != NGX_INVALID_FILE) {

        ngx_pool_run_cleanup_file(r->pool, r->cache->file.fd);

        r->cache->file.fd = NGX_INVALID_FILE;

    }

    switch (ngx_http_test_predicates(r, u->conf->no_cache)) {

    case NGX_ERROR:

        ngx_http_upstream_finalize_request(r, u, NGX_ERROR);

        return;

    case NGX_DECLINED:

        u->cacheable = 0;

        break;

    default: /* NGX_OK */

        if (u->cache_status == NGX_HTTP_CACHE_BYPASS) {

            /* create cache if previously bypassed */

            if (ngx_http_file_cache_create(r) != NGX_OK) {

                ngx_http_upstream_finalize_request(r, u, NGX_ERROR);

                return;

            }

        }

        break;

    }

    if (u->cacheable) {

        time_t  now, valid;

        now = ngx_time();

        valid = r->cache->valid_sec;

        if (valid == 0) {

            valid = ngx_http_file_cache_valid(u->conf->cache_valid,

                                              u->headers_in.status_n);

            if (valid) {

                r->cache->valid_sec = now + valid;

            }

        }

        if (valid) {

            r->cache->date = now;

            r->cache->body_start = (u_short) (u->buffer.pos - u->buffer.start);

            if (u->headers_in.status_n == NGX_HTTP_OK

                || u->headers_in.status_n == NGX_HTTP_PARTIAL_CONTENT)

            {

                r->cache->last_modified = u->headers_in.last_modified_time;

                if (u->headers_in.etag) {

                    r->cache->etag = u->headers_in.etag->value;

                } else {

                    ngx_str_null(&r->cache->etag);

                }

            } else {

                r->cache->last_modified = -1;

                ngx_str_null(&r->cache->etag);

            }

            if (ngx_http_file_cache_set_header(r, u->buffer.start) != NGX_OK) {

                ngx_http_upstream_finalize_request(r, u, NGX_ERROR);

                return;

            }

        } else {

            u->cacheable = 0;

        }

    }

    ngx_log_debug1(NGX_LOG_DEBUG_HTTP, c->log, 0,

                   "http cacheable: %d", u->cacheable);

    if (u->cacheable == 0 && r->cache) {

        ngx_http_file_cache_free(r->cache, u->pipe->temp_file);

    }

    if (r->header_only && !u->cacheable && !u->store) {

        ngx_http_upstream_finalize_request(r, u, 0);

        return;

    }

#endif

    p = u->pipe;

    p->output_filter = ngx_http_upstream_output_filter;

    p->output_ctx = r;

    p->tag = u->output.tag;

    p->bufs = u->conf->bufs;

    p->busy_size = u->conf->busy_buffers_size;

    p->upstream = u->peer.connection;

    p->downstream = c;

    p->pool = r->pool;

    p->log = c->log;

    p->limit_rate = u->conf->limit_rate;

    p->start_sec = ngx_time();

    p->cacheable = u->cacheable || u->store;

    p->temp_file = ngx_pcalloc(r->pool, sizeof(ngx_temp_file_t));

    if (p->temp_file == NULL) {

        ngx_http_upstream_finalize_request(r, u, NGX_ERROR);

        return;

    }

    p->temp_file->file.fd = NGX_INVALID_FILE;

    p->temp_file->file.log = c->log;

    p->temp_file->path = u->conf->temp_path;

    p->temp_file->pool = r->pool;

    if (p->cacheable) {

        p->temp_file->persistent = 1;

#if (NGX_HTTP_CACHE)

        if (r->cache && !r->cache->file_cache->use_temp_path) {

            p->temp_file->path = r->cache->file_cache->path;

            p->temp_file->file.name = r->cache->file.name;

        }

#endif

    } else {

        p->temp_file->log_level = NGX_LOG_WARN;

        p->temp_file->warn = "an upstream response is buffered "

                             "to a temporary file";

    }

    p->max_temp_file_size = u->conf->max_temp_file_size;

    p->temp_file_write_size = u->conf->temp_file_write_size;

#if (NGX_THREADS)

    if (clcf->aio == NGX_HTTP_AIO_THREADS && clcf->aio_write) {

        p->thread_handler = ngx_http_upstream_thread_handler;

        p->thread_ctx = r;

    }

#endif

    p->preread_bufs = ngx_alloc_chain_link(r->pool);

    if (p->preread_bufs == NULL) {

        ngx_http_upstream_finalize_request(r, u, NGX_ERROR);

        return;

    }

    p->preread_bufs->buf = &u->buffer;

    p->preread_bufs->next = NULL;

    u->buffer.recycled = 1;

    p->preread_size = u->buffer.last - u->buffer.pos;

    if (u->cacheable) {

        p->buf_to_file = ngx_calloc_buf(r->pool);

        if (p->buf_to_file == NULL) {

            ngx_http_upstream_finalize_request(r, u, NGX_ERROR);

            return;

        }

        p->buf_to_file->start = u->buffer.start;

        p->buf_to_file->pos = u->buffer.start;

        p->buf_to_file->last = u->buffer.pos;

        p->buf_to_file->temporary = 1;

    }

    if (ngx_event_flags & NGX_USE_IOCP_EVENT) {

        /* the posted aio operation may corrupt a shadow buffer */

        p->single_buf = 1;

    }

    /* TODO: p->free_bufs = 0 if use ngx_create_chain_of_bufs() */

    p->free_bufs = 1;

    /*

     * event_pipe would do u->buffer.last += p->preread_size

     * as though these bytes were read

     */

    u->buffer.last = u->buffer.pos;

    if (u->conf->cyclic_temp_file) {

        /*

         * we need to disable the use of sendfile() if we use cyclic temp file

         * because the writing a new data may interfere with sendfile()

         * that uses the same kernel file pages (at least on FreeBSD)

         */

        p->cyclic_temp_file = 1;

        c->sendfile = 0;

    } else {

        p->cyclic_temp_file = 0;

    }

    p->read_timeout = u->conf->read_timeout;

    p->send_timeout = clcf->send_timeout;

    p->send_lowat = clcf->send_lowat;

    p->length = -1;

    if (u->input_filter_init

        && u->input_filter_init(p->input_ctx) != NGX_OK)

    {

        ngx_http_upstream_finalize_request(r, u, NGX_ERROR);

        return;

    }

    u->read_event_handler = ngx_http_upstream_process_upstream;

    r->write_event_handler = ngx_http_upstream_process_downstream;

    ngx_http_upstream_process_upstream(r, u);

}

// 发送客户端数据时处理

// http/ngx_http_upstream.c

static void

ngx_http_upstream_process_upstream(ngx_http_request_t *r,

    ngx_http_upstream_t *u)

{

    ngx_event_t       *rev;

    ngx_event_pipe_t  *p;

    ngx_connection_t  *c;

    c = u->peer.connection;

    p = u->pipe;

    rev = c->read;

    ngx_log_debug0(NGX_LOG_DEBUG_HTTP, c->log, 0,

                   "http upstream process upstream");

    c->log->action = "reading upstream";

    if (rev->timedout) {

        p->upstream_error = 1;

        ngx_connection_error(c, NGX_ETIMEDOUT, "upstream timed out");

    } else {

        if (rev->delayed) {

            ngx_log_debug0(NGX_LOG_DEBUG_HTTP, c->log, 0,

                           "http upstream delayed");

            if (ngx_handle_read_event(rev, 0) != NGX_OK) {

                ngx_http_upstream_finalize_request(r, u, NGX_ERROR);

            }

            return;

        }

        // 管道式读取数据响应数据，即不会一次性输出

        // 而是源源不断地输出

        if (ngx_event_pipe(p, 0) == NGX_ABORT) {

            ngx_http_upstream_finalize_request(r, u, NGX_ERROR);

            return;

        }

    }

    ngx_http_upstream_process_request(r, u);

}

// http/ngx_http_upstream.c

// 响应客户端

static void

ngx_http_upstream_process_request(ngx_http_request_t *r,

    ngx_http_upstream_t *u)

{

    ngx_temp_file_t   *tf;

    ngx_event_pipe_t  *p;

    p = u->pipe;

#if (NGX_THREADS)

    if (p->writing && !p->aio) {

        /*

         * make sure to call ngx_event_pipe()

         * if there is an incomplete aio write

         */

        if (ngx_event_pipe(p, 1) == NGX_ABORT) {

            ngx_http_upstream_finalize_request(r, u, NGX_ERROR);

            return;

        }

    }

    if (p->writing) {

        return;

    }

#endif

    if (u->peer.connection) {

        if (u->store) {

            if (p->upstream_eof || p->upstream_done) {

                tf = p->temp_file;

                if (u->headers_in.status_n == NGX_HTTP_OK

                    && (p->upstream_done || p->length == -1)

                    && (u->headers_in.content_length_n == -1

                        || u->headers_in.content_length_n == tf->offset))

                {

                    ngx_http_upstream_store(r, u);

                }

            }

        }

#if (NGX_HTTP_CACHE)

        if (u->cacheable) {

            if (p->upstream_done) {

                ngx_http_file_cache_update(r, p->temp_file);

            } else if (p->upstream_eof) {

                tf = p->temp_file;

                if (p->length == -1

                    && (u->headers_in.content_length_n == -1

                        || u->headers_in.content_length_n

                           == tf->offset - (off_t) r->cache->body_start))

                {

                    ngx_http_file_cache_update(r, tf);

                } else {

                    ngx_http_file_cache_free(r->cache, tf);

                }

            } else if (p->upstream_error) {

                ngx_http_file_cache_free(r->cache, p->temp_file);

            }

        }

#endif

        if (p->upstream_done || p->upstream_eof || p->upstream_error) {

            ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,

                           "http upstream exit: %p", p->out);

            // 输出完成，关闭连接

            // 没有输出完成的情况，会等待下一次就绪事件，再进行处理

            // 而无需等待目标服务器完全响应，再返回本次处理

            // 这就是非阻塞的优势所在

            if (p->upstream_done

                || (p->upstream_eof && p->length == -1))

            {

                // 关闭连接

                ngx_http_upstream_finalize_request(r, u, 0);

                return;

            }

            if (p->upstream_eof) {

                ngx_log_error(NGX_LOG_ERR, r->connection->log, 0,

                              "upstream prematurely closed connection");

            }

            ngx_http_upstream_finalize_request(r, u, NGX_HTTP_BAD_GATEWAY);

            return;

        }

    }

    if (p->downstream_error) {

        ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,

                       "http upstream downstream error");

        if (!u->cacheable && !u->store && u->peer.connection) {

            ngx_http_upstream_finalize_request(r, u, NGX_ERROR);

        }

    }

}