Changeset 1502

Timestamp:

01/05/2007 04:43:40 AM (22 months ago)

Author:

jakabosky

Message:

Moved HTTP/1.1 chunk encoding into plugin mod_chunked
Created filter chain API for mod_chunked and mod_deflate.
Added new handle_filter_response_content hook to plugin API.

Location:

trunk

Files:

• configure.in (modified) (1 diff)
• src/Makefile.am (modified) (3 diffs)
• src/base.h (modified) (2 diffs)
• src/chunk.c (modified) (1 diff)
• src/chunk.h (modified) (1 diff)
• src/configfile.c (modified) (2 diffs)
• src/connections.c (modified) (15 diffs)
• src/filter.c (added)
• src/filter.h (added)
• src/mod_chunked.c (added)
• src/mod_compress.c (modified) (2 diffs)
• src/mod_proxy_core.c (modified) (2 diffs)
• src/mod_proxy_core.h (modified) (1 diff)
• src/mod_proxy_core_protocol.h (modified) (1 diff)
• src/plugin.c (modified) (5 diffs)
• src/plugin.h (modified) (2 diffs)

trunk/configure.in

@@ -569 +569 @@
 
 
-do_build="mod_proxy_core mod_proxy_backend_http mod_proxy_backend_fastcgi mod_proxy_backend_scgi mod_evhost mod_simple_vhost mod_access mod_alias mod_setenv mod_usertrack mod_auth mod_status mod_accesslog mod_rrdtool mod_secdownload mod_expire mod_compress mod_dirlisting mod_indexfiles mod_userdir mod_webdav mod_staticfile mod_flv_streaming" 
+do_build="mod_proxy_core mod_proxy_backend_http mod_proxy_backend_fastcgi mod_proxy_backend_scgi mod_evhost mod_simple_vhost mod_access mod_alias mod_setenv mod_usertrack mod_auth mod_status mod_accesslog mod_rrdtool mod_secdownload mod_expire mod_compress mod_dirlisting mod_indexfiles mod_userdir mod_webdav mod_staticfile mod_chunked mod_flv_streaming" 
 
 plugins="mod_rewrite mod_redirect mod_ssi mod_trigger_b4_dl"

trunk/src/Makefile.am

@@ -49 +49 @@
 
 common_src=buffer.c log.c \
-      keyvalue.c chunk.c  \
+      keyvalue.c chunk.c filter.c \
       stream.c fdevent.c \
       stat_cache.c plugin.c joblist.c etag.c array.c \
@@ -142 +142 @@
 mod_staticfile_la_LIBADD = $(common_libadd)
 
+lib_LTLIBRARIES += mod_chunked.la
+mod_chunked_la_SOURCES = mod_chunked.c 
+mod_chunked_la_LDFLAGS = -module -export-dynamic -avoid-version -no-undefined
+mod_chunked_la_LIBADD = $(common_libadd)
+
 lib_LTLIBRARIES += mod_dirlisting.la
 mod_dirlisting_la_SOURCES = mod_dirlisting.c 
@@ -273 +278 @@
 
 hdr = server.h buffer.h network.h log.h keyvalue.h \
-      response.h request.h fastcgi.h chunk.h \
+      response.h request.h fastcgi.h chunk.h filter.h \
       settings.h http_auth_digest.h \
       md5.h http_auth.h stream.h \

trunk/src/base.h

@@ -20 +20 @@
 #include "array.h"
 #include "chunk.h"
+#include "filter.h"
 #include "keyvalue.h"
 #include "settings.h"
@@ -331 +332 @@
         int file_started;
 
-        chunkqueue *send;            /* the response-content without encoding */
+        chunkqueue *send;            /* the response-content before filters are applied */
         chunkqueue *recv;            /* the request-content, without encoding */
 
+        filter_chain *send_filters;  /* the chain of filters to apply to response-content. */
         chunkqueue *send_raw;        /* the full response (HTTP-Header + compression + chunking ) */
         chunkqueue *recv_raw;        /* the full request (HTTP-Header + chunking ) */

trunk/src/chunk.c

@@ -242 +242 @@
 
 /*
+ * copy/steal all chunks from in chunkqueue.  return total bytes copied/stolen.
+ *
+ */
+int chunkqueue_steal_all_chunks(chunkqueue *cq, chunkqueue *in) {
+        size_t total = 0;
+        off_t we_have = 0;
+        chunk *c;
+
+        if (!cq || !in) return 0;
+
+        for (c = in->first; c; c = c->next) {
+                switch (c->type) {
+                case MEM_CHUNK:
+                        if (c->mem->used == 0) continue;
+
+                        we_have = c->mem->used - c->offset - 1;
+                        if(we_have == 0) continue;
+                        if (c->offset == 0) {
+                                chunkqueue_steal_chunk(cq, c);
+                        } else {
+                                chunkqueue_append_buffer(cq, c->mem);
+                                c->offset = c->mem->used - 1;
+                        }
+                        break;
+                case FILE_CHUNK:
+                        if (c->file.length == 0) continue;
+
+                        we_have = c->file.length;
+                        if(c->file.is_temp) {
+                                chunkqueue_steal_tempfile(cq, c);
+                        } else {
+                                chunkqueue_append_file(cq, c->file.name, c->file.start, c->file.length);
+                        }
+
+                        c->offset = c->file.length;
+                        break;
+                case UNUSED_CHUNK:
+                        break;
+                }
+                total += we_have;
+        }
+
+        return total;
+}
+
+/*
  * copy/steal max_len bytes from chunk chain.  return total bytes copied/stolen.
  *

trunk/src/chunk.h

@@ -68 +68 @@
 int chunkqueue_steal_chunk(chunkqueue *cq, chunk *c);
 int chunkqueue_steal_chunks_len(chunkqueue *cq, chunk *c, size_t max_len);
+int chunkqueue_steal_all_chunks(chunkqueue *cq, chunkqueue *in);
 int chunkqueue_skip(chunkqueue *cq, off_t skip);
 void chunkqueue_remove_empty_last_chunk(chunkqueue *cq);

trunk/src/configfile.c

@@ -1020 +1020 @@
                         array_insert_unique(modules->value, (data_unset *)ds);
                 }
+
+                if (NULL == array_get_element(modules->value, "mod_chunked")) {
+                        ds = data_string_init();
+                        buffer_copy_string(ds->value, "mod_chunked");
+                        array_insert_unique(modules->value, (data_unset *)ds);
+                }
         } else {
                 data_string *ds;
@@ -1036 +1042 @@
                 ds = data_string_init();
                 buffer_copy_string(ds->value, "mod_staticfile");
+                array_insert_unique(modules->value, (data_unset *)ds);
+
+                ds = data_string_init();
+                buffer_copy_string(ds->value, "mod_chunked");
                 array_insert_unique(modules->value, (data_unset *)ds);
 

trunk/src/connections.c

@@ -184 +184 @@
 
 static int connection_handle_response_header(server *srv, connection *con) {
-        data_string *cl;
+        int no_response_body = 0;
 
         if (con->mode == DIRECT) {
@@ -214 +214 @@
 
                                 /* trash the content */
-                                chunkqueue_reset(con->send);
+                                no_response_body = 1;
 
                                 con->http_status = 200;
-                                con->send->is_closed = 1;
-
                         }
                         break;
@@ -331 +329 @@
         case 304:
         default:
+                no_response_body = 1;
+                break;
+        }
+
+        if (con->request.http_method == HTTP_METHOD_HEAD) {
+                no_response_body = 1;
+        }
+
+        if (no_response_body) {
                 /* disable chunked encoding again as we have no body */
                 con->response.transfer_encoding &= ~HTTP_TRANSFER_ENCODING_CHUNKED;
@@ -336 +343 @@
 
                 con->send->is_closed = 1;
-
-                break;
-        }
-
-
-        if (con->send->is_closed) {
-                /* we have all the content and chunked encoding is not used, set a content-length */
-
-                if ((con->response.transfer_encoding & HTTP_TRANSFER_ENCODING_CHUNKED) == 0 &&
-                    NULL == (cl = (data_string *)array_get_element(con->response.headers, "Content-Length"))) {
-                        buffer_copy_off_t(srv->tmp_buf, chunkqueue_length(con->send));
-
-                        response_header_overwrite(srv, con, CONST_STR_LEN("Content-Length"), CONST_BUF_LEN(srv->tmp_buf));
-                }
-        } else {
-                if (NULL == (cl = (data_string *)array_get_element(con->response.headers, "Content-Length"))) {
-                        /* we don't know the size of the content yet
-                         * - either enable chunking
-                         * - or disable keep-alive  */
-
-                        if (con->request.http_version == HTTP_VERSION_1_1) {
-                                /* enable chunk-encoding */
-                                con->response.transfer_encoding |= HTTP_TRANSFER_ENCODING_CHUNKED;
-                        } else  {
-                                con->keep_alive = 0;
-                        }
-                }
-        }
-
-        if (con->request.http_method == HTTP_METHOD_HEAD) {
-                con->response.transfer_encoding &= ~HTTP_TRANSFER_ENCODING_CHUNKED;
-                chunkqueue_reset(con->send);
-
-                con->send->is_closed = 1;
-        }
-
-        http_response_write_header(srv, con, con->send_raw);
+        }
 
         return 0;
@@ -420 +391 @@
 
 #undef CLEAN
-        con->send = chunkqueue_init();
+        con->send_filters = filter_chain_init();
+        /* send is the chunkqueue of the first send filter */
+        con->send = con->send_filters->first->cq;
         con->recv = chunkqueue_init();
 
@@ -453 +426 @@
                 iosocket_free(con->sock);
 
-                chunkqueue_free(con->send);
+                filter_chain_free(con->send_filters);
+                con->send = NULL;
                 chunkqueue_free(con->recv);
                 chunkqueue_free(con->send_raw);
@@ -566 +540 @@
         array_reset(con->environment);
 
-        chunkqueue_reset(con->send);
+        filter_chain_reset(con->send_filters);
+        con->send = con->send_filters->first->cq;
         chunkqueue_reset(con->recv);
         chunkqueue_reset(con->send_raw);
@@ -939 +914 @@
 }
 
-static int http_chunk_append_len(chunkqueue *cq, size_t len) {
-        size_t i, olen = len, j;
-        buffer *b;
-
-        b = buffer_init();
-
-        if (len == 0) {
-                buffer_copy_string(b, "0");
-        } else {
-                for (i = 0; i < 8 && len; i++) {
-                        len >>= 4;
-                }
-
-                /* i is the number of hex digits we have */
-                buffer_prepare_copy(b, i + 1);
-
-                for (j = i-1, len = olen; j+1 > 0; j--) {
-                        b->ptr[j] = (len & 0xf) + (((len & 0xf) <= 9) ? '0' : 'a' - 10);
-                        len >>= 4;
-                }
-                b->used = i;
-                b->ptr[b->used++] = '\0';
-        }
-
-        buffer_append_string(b, "\r\n");
-        chunkqueue_append_buffer(cq, b);
-        len = b->used - 1;
-
-        buffer_free(b);
-
-        return len;
-}
-
-
-/**
- * apply chunk encoding if necessary
- */
-int http_stream_encoder(server *srv, connection *con, chunkqueue *in, chunkqueue *out) {
-        chunk *c;
-        int is_chunked;
-        int we_have = 0;
-
-        UNUSED(srv);
-
-        /* no more data to encode. */
-        if (out->is_closed) return 0;
-        /**/
-
-        is_chunked = (con->response.transfer_encoding & HTTP_TRANSFER_ENCODING_CHUNKED);
-
-        if (is_chunked) {
-                for (c = in->first; c; c = c->next) {
-                        switch (c->type) {
-                        case MEM_CHUNK:
-                                if (c->mem->used == 0) continue;
-
-                                we_have = c->mem->used - c->offset - 1;
-                                in->bytes_out += we_have;
-                                if(we_have == 0) continue;
-                                we_have += http_chunk_append_len(out, we_have);
-                                chunkqueue_append_buffer(out, c->mem);
-                                c->offset = c->mem->used - 1;
-                                break;
-                        case FILE_CHUNK:
-                                if (c->file.length == 0) continue;
-
-                                we_have = c->file.length;
-                                in->bytes_out += we_have;
-                                we_have += http_chunk_append_len(out, c->file.length);
-                                if(c->file.is_temp) {
-                                        chunkqueue_steal_tempfile(out, c);
-                                } else {
-                                        chunkqueue_append_file(out, c->file.name, c->file.start, c->file.length);
-                                }
-
-                                c->offset = c->file.length;
-                                break;
-                        case UNUSED_CHUNK:
-                                break;
-                        }
-                        chunkqueue_append_mem(out, "\r\n", 2 + 1);
-                        we_have += 2;
-                        out->bytes_in += we_have;
-                }
-                if (in->is_closed) {
-                        chunkqueue_append_mem(out, "0\r\n\r\n", 5 + 1);
-                        out->bytes_in += 5;
-                }
-        } else {
-                for (c = in->first; c; c = c->next) {
-                        switch (c->type) {
-                        case MEM_CHUNK:
-                                if (c->mem->used == 0) continue;
-
-                                we_have = c->mem->used - c->offset - 1;
-                                in->bytes_out += we_have;
-                                if(we_have == 0) continue;
-                                if (c->offset == 0) {
-                                        chunkqueue_steal_chunk(out, c);
-                                } else {
-                                        chunkqueue_append_buffer(out, c->mem);
-                                        c->offset = c->mem->used - 1;
-                                }
-                                break;
-                        case FILE_CHUNK:
-                                if (c->file.length == 0) continue;
-
-                                we_have = c->file.length;
-                                in->bytes_out += we_have;
-                                if(c->file.is_temp) {
-                                        chunkqueue_steal_tempfile(out, c);
-                                } else {
-                                        chunkqueue_append_file(out, c->file.name, c->file.start, c->file.length);
-                                }
-
-                                c->offset = c->file.length;
-                                break;
-                        case UNUSED_CHUNK:
-                                break;
-                        }
-                        in->bytes_out += we_have;
-                        out->bytes_in += we_have;
-                }
-        }
-
-        chunkqueue_remove_finished_chunks(in);
-
-        if (in->is_closed) {
-                /* mark the output queue as finished. */
-                out->is_closed = 1;
-        }
-        return 0;
-}
-
 int connection_state_machine(server *srv, connection *con) {
         int done = 0, r;
@@ -1296 +1137 @@
                             con->request.content_length == 0) {
                                 con->recv->is_closed = 1;
-                        }
+                        }
 
                         if (!con->recv->is_closed &&
@@ -1356 +1197 @@
                         break;
                 case CON_STATE_HANDLE_RESPONSE_HEADER:
+                        /* handle the HTTP response headers, or generate error-page */
+                        connection_handle_response_header(srv, con);
+
                         /* we got a response header from the backend
                          * call all plugins who want to modify the response header
@@ -1362 +1206 @@
                          *
                          */
+                        switch (plugins_call_handle_response_header(srv, con)) {
+                        case HANDLER_GO_ON:
+                        default:
+                                break;
+                        }
 
                         connection_set_state(srv, con, CON_STATE_WRITE_RESPONSE_HEADER);
@@ -1367 +1216 @@
                         break;
                 case CON_STATE_WRITE_RESPONSE_HEADER:
-                        /* create the HTTP response header */
-                        connection_handle_response_header(srv, con);
+                        /* write response headers */
+                        http_response_write_header(srv, con, con->send_raw);
 
                         connection_set_state(srv, con, CON_STATE_WRITE_RESPONSE_CONTENT);
@@ -1375 +1224 @@
                 case CON_STATE_WRITE_RESPONSE_CONTENT:
                         fdevent_event_del(srv->ev, con->sock);
+
+                        /* looks like we shall read some content from the backend */
+                        switch (plugins_call_handle_read_response_content(srv, con)) {
+                        case HANDLER_WAIT_FOR_EVENT:
+                                if (!con->send->is_closed && con->send->bytes_in == con->send->bytes_out) {
+                                        /* need to wait for more data */
+                                        return HANDLER_WAIT_FOR_EVENT;
+                                }
+                                break;
+                        case HANDLER_GO_ON:
+                        default:
+                                break;
+                        }
 
                         /* we might have new content in the con->send buffer
@@ -1381 +1243 @@
                          * - compression
                          */
-
-                        /* looks like we shall read some content from the backend */
-
-                        switch (plugins_call_handle_read_response_content(srv, con)) {
+                        switch (plugins_call_handle_filter_response_content(srv, con)) {
                         case HANDLER_GO_ON:
                         default:
@@ -1390 +1249 @@
                         }
 
-                        http_stream_encoder(srv, con, con->send, con->send_raw);
-
+                        /* copy output from filters into send_raw. */
+                        r = filter_chain_copy_output(con->send_filters, con->send_raw);
+
+                        if (!con->send_raw->is_closed && con->send_raw->bytes_in == con->send_raw->bytes_out) {
+                                return HANDLER_WAIT_FOR_EVENT;
+                        }
                         switch(network_write_chunkqueue(srv, con, con->send_raw)) {
                         case NETWORK_STATUS_SUCCESS:

trunk/src/mod_compress.c

@@ -424 +424 @@
                 /* file exists */
 
-                chunkqueue_append_file(con->send_raw, p->ofn, 0, compressed_sce->st.st_size);
+                chunkqueue_append_file(con->send, p->ofn, 0, compressed_sce->st.st_size);
                 con->send->is_closed = 1;
 
@@ -500 +500 @@
         if (ret != 0) return -1;
 
-        chunkqueue_append_file(con->send_raw, p->ofn, 0, r);
+        chunkqueue_append_file(con->send, p->ofn, 0, r);
         con->send->is_closed = 1;
 

trunk/src/mod_proxy_core.c

@@ -698 +698 @@
         }
 
-        /* we are finished decoding the response content. */
-        if(out->is_closed) {
-                proxy_copy_response(srv, con, sess);
-        } else {
+        /* we are finished decoding the response headers. */
+        if(!out->is_closed) {
                 /* We don't have all the response content try to enable chunked encoding. */
                 /* does the client allow us to send chunked encoding ? */
@@ -709 +707 @@
                 }
         }
+
+        /* we might have part of the response content too */
+        proxy_copy_response(srv, con, sess);
 
         return PARSE_SUCCESS; /* we have a full header */

trunk/src/mod_proxy_core.h

@@ -67 +67 @@
 } proxy_state_t;
 
-typedef struct {
+typedef struct proxy_session {
         proxy_connection *proxy_con;
         proxy_backend *proxy_backend;

trunk/src/mod_proxy_core_protocol.h

@@ -12 +12 @@
                 static int x(server *srv, proxy_session *sess, chunkqueue *in, chunkqueue *out)
 
-typedef struct {
+typedef struct proxy_protocol {
         buffer *name;
 

trunk/src/plugin.c

@@ -41 +41 @@
                 PLUGIN_FUNC_HANDLE_START_BACKEND,
                 PLUGIN_FUNC_HANDLE_SEND_REQUEST_CONTENT,
+                PLUGIN_FUNC_HANDLE_RESPONSE_HEADER,
                 PLUGIN_FUNC_HANDLE_READ_RESPONSE_CONTENT,
+                PLUGIN_FUNC_HANDLE_FILTER_RESPONSE_CONTENT,
                 PLUGIN_FUNC_HANDLE_JOBLIST,
                 PLUGIN_FUNC_HANDLE_DOCROOT,
@@ -122 +124 @@
 PLUGIN_STATIC(mod_evhost);
 PLUGIN_STATIC(mod_expire);
+PLUGIN_STATIC(mod_chunked);
 PLUGIN_STATIC(mod_indexfile);
 PLUGIN_STATIC(mod_mysql_vhost);
@@ -157 +160 @@
 PLUGIN_STATIC(mod_evhost),
 PLUGIN_STATIC(mod_expire),
+PLUGIN_STATIC(mod_chunked),
 PLUGIN_STATIC(mod_indexfile),
 PLUGIN_STATIC(mod_mysql_vhost),
@@ -366 +370 @@
 PLUGIN_TO_SLOT(PLUGIN_FUNC_HANDLE_START_BACKEND, handle_start_backend)
 PLUGIN_TO_SLOT(PLUGIN_FUNC_HANDLE_SEND_REQUEST_CONTENT, handle_send_request_content)
+PLUGIN_TO_SLOT(PLUGIN_FUNC_HANDLE_RESPONSE_HEADER, handle_response_header)
 PLUGIN_TO_SLOT(PLUGIN_FUNC_HANDLE_READ_RESPONSE_CONTENT, handle_read_response_content)
+PLUGIN_TO_SLOT(PLUGIN_FUNC_HANDLE_FILTER_RESPONSE_CONTENT, handle_filter_response_content)
 PLUGIN_TO_SLOT(PLUGIN_FUNC_HANDLE_CONNECTION_CLOSE, handle_connection_close)
 PLUGIN_TO_SLOT(PLUGIN_FUNC_HANDLE_RESPONSE_DONE, handle_response_done)
@@ -494 +500 @@
                 PLUGIN_TO_SLOT(PLUGIN_FUNC_HANDLE_START_BACKEND, handle_start_backend);
                 PLUGIN_TO_SLOT(PLUGIN_FUNC_HANDLE_SEND_REQUEST_CONTENT, handle_send_request_content);
+                PLUGIN_TO_SLOT(PLUGIN_FUNC_HANDLE_RESPONSE_HEADER, handle_response_header);
                 PLUGIN_TO_SLOT(PLUGIN_FUNC_HANDLE_READ_RESPONSE_CONTENT, handle_read_res