2009-04-05

correcting Transfer-encoding:chunked problem in nginx

While Igor Sysoev is thinking on how to properly (i.e. robustly and efficiently) implement handling of Content-encoding:chunked with unset Content-Length in nginx, I needed to hack a quick workaround (for those, who need it "here and now"). I decided to post it here for those who may need it:

--- old/ngx_http_request.c 2009-04-05 20:41:18.000000000 +0300
+++ new/ngx_http_request.c 2009-04-05 20:38:33.000000000 +0300
@@ -1403,16 +1403,6 @@
         return NGX_ERROR;
     }
 
-    if (r->headers_in.transfer_encoding
-        && ngx_strcasestrn(r->headers_in.transfer_encoding->value.data,
-                           "chunked", 7 - 1))
-    {
-        ngx_log_error(NGX_LOG_INFO, r->connection->log, 0,
-                      "client sent \"Transfer-Encoding: chunked\" header");
-        ngx_http_finalize_request(r, NGX_HTTP_LENGTH_REQUIRED);
-        return NGX_ERROR;
-    }
-
     if (r->headers_in.connection_type == NGX_HTTP_CONNECTION_KEEP_ALIVE) {
         if (r->headers_in.keep_alive) {
             r->headers_in.keep_alive_n =


--- old/ngx_http_request_body.c 2009-04-05 20:41:39.000000000 +0300
+++ new/ngx_http_request_body.c 2009-04-05 20:34:45.000000000 +0300
@@ -54,11 +54,6 @@
 
     r->request_body = rb;
 
-    if (r->headers_in.content_length_n < 0) {
-        post_handler(r);
-        return NGX_OK;
-    }
-
     clcf = ngx_http_get_module_loc_conf(r, ngx_http_core_module);
 
     if (r->headers_in.content_length_n == 0) {
@@ -180,7 +175,8 @@
 
     } else {
         b = NULL;
-        rb->rest = r->headers_in.content_length_n;
+        rb->rest = r->headers_in.content_length_n >= 0 ? r->headers_in.content_length_n
+                                                       : NGX_MAX_SIZE_T_VALUE;
         next = &rb->bufs;
     }


In the mean time I've sorted out for myself the server's internals. What can I say? First of all the event-driven architecture is complicated. Especially, when there's no high-level description, who is calling what. And especially under virtually complete absence of comments in the code.

And about C and Lisp (based on examining the source of the two HTTP-servers: nginx and Hunchentoot). Lisp is clay, shape whatever you want with all the connected advantages and shortcomings. C... There's a children's construction set from metal bars and corners. It's fun to play with them. And, generally, everything is quite clear, it's possible to assemble good, solid constructions. But making a vase with rounded corners — it's for the geeks...

1 comment:

  1. Vsevolod DyomkinApril 9, 2009 at 12:10 PM

    Alas, this solution was a premature one: turned out, that it relied on nginx handling of preread data, which worked only on fast connections (for example, in my testing environment on localhost :).

    Below is the more adequate one:
    --- old/ngx_http_request_body.c 2009-04-05 20:44:46.000000000 +0300
    +++ new/ngx_http_request_body.c 2009-04-09 12:03:56.000000000 +0300
    @@ -54,7 +54,7 @@

    r->request_body = rb;

    - if (r->headers_in.content_length_n < 0) {
    + if (r->headers_in.content_length_n < 0 && !(r->method & NGX_HTTP_POST)) {
    post_handler(r);
    return NGX_OK;
    }
    @@ -137,7 +137,8 @@

    rb->buf = b;

    - if ((off_t) preread >= r->headers_in.content_length_n) {
    + if (r->headers_in.content_length_n >= 0 &&
    + (off_t) preread >= r->headers_in.content_length_n) {

    /* the whole request body was pre-read */

    @@ -163,9 +164,12 @@

    r->request_length += preread;

    - rb->rest = r->headers_in.content_length_n - preread;
    + if (r->headers_in.content_length_n >= 0) {
    + rb->rest = r->headers_in.content_length_n - preread;
    + }

    - if (rb->rest <= (off_t) (b->end - b->last)) {
    + if (r->headers_in.content_length_n >= 0 &&
    + rb->rest <= (off_t) (b->end - b->last)) {

    /* the whole request body may be placed in r->header_in */

    @@ -180,14 +184,14 @@

    } else {
    b = NULL;
    - rb->rest = r->headers_in.content_length_n;
    + rb->rest = r->headers_in.content_length_n >= 0 ?r->headers_in.content_length_n : 1024;
    next = &rb->bufs;
    }

    size = clcf->client_body_buffer_size;
    size += size >> 2;

    - if (rb->rest < size) {
    + if (rb->rest >=0 && rb->rest < size) {
    size = (ssize_t) rb->rest;

    if (r->request_body_in_single_buf) {
    @@ -224,13 +228,15 @@

    *next = cl;

    - if (r->request_body_in_file_only || r->request_body_in_single_buf) {
    + if (r->request_body_in_file_only || r->request_body_in_single_buf || r->headers_in.content_length_n < 0) {
    rb->to_write = rb->bufs;

    } else {
    rb->to_write = rb->bufs->next ? rb->bufs->next : rb->bufs;
    }

    + // if (r->headers_in.content_length_n < 0) {
    +
    r->read_event_handler = ngx_http_read_client_request_body_handler;

    return ngx_http_do_read_client_request_body(r);
    @@ -310,7 +316,13 @@
    }

    rb->buf->last += n;
    - rb->rest -= n;
    +
    + if ( ngx_memcmp(rb->buf->last - 7, (u_char *) CRLF "0" CRLF CRLF, sizeof(u_char)) ) {
    + rb->rest -= n;
    + } else {
    + rb->rest = 0;
    + }
    +
    r->request_length += n;

    if (rb->rest == 0) {

    ReplyDelete