Changeset 696

Timestamp:

09/08/2005 10:00:32 AM (3 years ago)

Author:

jan

Message:

- don't keep the full fastcgi response in memory
- don't cache more than 4 chunks in a queue for reuse
- on chunkqueue_reset, clean the chunks

Location:

branches/lighttpd-merge-1.4.x/src

Files:

• chunk.c (modified) (4 diffs)
• chunk.h (modified) (2 diffs)
• mod_fastcgi.c (modified) (12 diffs)
• network.c (modified) (1 diff)

branches/lighttpd-merge-1.4.x/src/chunk.c

@@ -55 +55 @@
         free(c);
 }
+
+static void chunk_reset(chunk *c) {
+        if (!c) return;
+        
+        /* c->data.mem overlaps with c->data.file.name */
+        switch (c->type) {
+        case MEM_CHUNK: buffer_reset(c->data.mem); break;
+        case FILE_CHUNK: buffer_reset(c->data.file.name); break;
+        default: break; 
+        }
+}
+
 
 void chunkqueue_free(chunkqueue *cq) {
@@ -87 +99 @@
                 cq->unused = c->next;
                 c->next = NULL;
+                cq->unused_chunks--;
         }
         
@@ -117 +130 @@
 
 void chunkqueue_reset(chunkqueue *cq) {
+        chunk *c;
         /* move everything to the unused queue */
         
-        if (cq->last == NULL) return;
-        
-        cq->last->next = cq->unused;
-        cq->unused = cq->first;
-        
-        /* disconnect active chain */
-        cq->first = cq->last = NULL;
+        /* mark all read written */ 
+        for (c = cq->first; c; c = c->next) {
+                switch(c->type) {
+                case MEM_CHUNK:
+                        c->offset = c->data.mem->used - 1;
+                        break;
+                case FILE_CHUNK:
+                        c->offset = c->data.file.length;
+                        break;
+                default: 
+                        break;
+                }
+        }
+
+        chunkqueue_remove_finished_chunks(cq);
 }
 
@@ -262 +284 @@
 }
 
-#ifdef DEBUG_CHUNK
-
-static int write_chunkqueue(int fd, chunkqueue *c) {
-        UNUSED(fd);
-        UNUSED(c);
-
-        return 0;
-}
-
-int main(int argc, char **argv) {
-        chunkqueue *c;
-        buffer *b, *fn;
-        
-        UNUSED(argc);
-        UNUSED(argv);
-
-        c = chunkqueue_init();
-        
-        fn = buffer_init_string("server.c");
-        
-        chunkqueue_append_file(c, fn, 0, 10);
-        chunkqueue_append_file(c, fn, 10, 10);
-        chunkqueue_append_file(c, fn, 20, 10);
-        
-        write_chunkqueue(STDERR_FILENO, c);
-        chunkqueue_reset(c);
-        
-        b = buffer_init();
-        buffer_copy_string(b, "\ntest string mit vielen Zeichen\n");
-        chunkqueue_append_buffer(c, b);
-        
-        write_chunkqueue(STDERR_FILENO, c);
-        chunkqueue_reset(c);
-        
-        chunkqueue_append_file(c, fn, 0, 10);
-        buffer_copy_string(b, "\ntest string mit vielen Zeichen\n");
-        chunkqueue_append_buffer(c, b);
-        chunkqueue_append_file(c, fn, 10, 10);
-        chunkqueue_append_file(c, fn, 20, 10);
-        chunkqueue_append_file(c, fn, 50, 40);
-        
-        write_chunkqueue(STDERR_FILENO, c);
-        chunkqueue_reset(c);
-        
-        chunkqueue_free(c);
-        
-        return 0;
-}
-#endif
+int chunkqueue_remove_finished_chunks(chunkqueue *cq) {
+        chunk *c;
+
+        for (c = cq->first; c; c = cq->first) {
+                int is_finished = 0;
+
+                switch (c->type) {
+                case MEM_CHUNK:
+                        if (c->offset == (off_t)c->data.mem->used - 1) is_finished = 1;
+                        break;
+                case FILE_CHUNK:
+                        if (c->offset == c->data.file.length) is_finished = 1; 
+                        break;
+                default: 
+                        break;
+                }
+
+                if (!is_finished) break;
+
+                chunk_reset(c);
+
+                cq->first = c->next;
+                if (c == cq->last) cq->last = NULL;
+
+                /* keep at max 4 chunks in the 'unused'-cache */
+                if (cq->unused_chunks > 4) {
+                        chunk_free(c);
+                } else {
+                        c->next = cq->unused;
+                        cq->unused = c;
+                        cq->unused_chunks++;
+                }
+        }
+
+        return 0;
+}

branches/lighttpd-merge-1.4.x/src/chunk.h

@@ -36 +36 @@
         
         chunk *unused;
+        size_t unused_chunks;
 } chunkqueue;
 
@@ -47 +48 @@
 buffer * chunkqueue_get_prepend_buffer(chunkqueue *c);
 
+int chunkqueue_remove_finished_chunks(chunkqueue *cq);
+
 off_t chunkqueue_length(chunkqueue *c);
 off_t chunkqueue_written(chunkqueue *c);

branches/lighttpd-merge-1.4.x/src/mod_fastcgi.c

@@ -303 +303 @@
 
 typedef struct {
-        buffer  *response; 
-        size_t   response_len;
-        int      response_type;
-        int      response_padding;
-        size_t   response_request_id;
-        
         fcgi_proc *proc;
         fcgi_extension_host *host;
@@ -320 +314 @@
         size_t    write_offset;
         
-        read_buffer *rb;
+        chunkqueue *rb;
         
         buffer   *response_header;
@@ -355 +349 @@
         hctx->fde_ndx = -1;
         
-        hctx->response = buffer_init();
         hctx->response_header = buffer_init();
         hctx->write_buffer = buffer_init();
@@ -363 +356 @@
         hctx->proc = NULL;
         
-        hctx->response_len = 0;
-        hctx->response_type = 0;
-        hctx->response_padding = 0;
-        hctx->response_request_id = 0;
         hctx->fd = -1;
         
         hctx->reconnects = 0;
+
+        hctx->rb = chunkqueue_init();
         
         return hctx;
@@ -375 +366 @@
 
 static void handler_ctx_free(handler_ctx *hctx) {
-        buffer_free(hctx->response);
         buffer_free(hctx->response_header);
         buffer_free(hctx->write_buffer);
-        
-        if (hctx->rb) {
-                if (hctx->rb->ptr) free(hctx->rb->ptr);
-                free(hctx->rb);
-        }
-        
+
+        chunkqueue_free(hctx->rb);
+
         free(hctx);
 }
@@ -1940 +1927 @@
 }
 
+typedef struct {
+        buffer  *b; 
+        size_t   len;
+        int      type;
+        int      padding;
+        size_t   request_id;
+} fastcgi_response_packet;
+
+static int fastcgi_get_packet(server *srv, handler_ctx *hctx, fastcgi_response_packet *packet) {
+        chunk * c;
+        size_t offset = 0;
+        size_t toread = 0;
+        FCGI_Header *header;
+
+        if (!hctx->rb->first) return -1;
+
+        packet->b = buffer_init();
+        packet->len = 0;
+        packet->type = 0;
+        packet->padding = 0;
+        packet->request_id = 0;
+
+        /* get at least the FastCGI header */
+        for (c = hctx->rb->first; c; c = c->next) {
+                if (packet->b->used == 0) {
+                        buffer_copy_string_len(packet->b, c->data.mem->ptr + c->offset, c->data.mem->used - c->offset - 1);
+                } else {
+                        buffer_append_string_len(packet->b, c->data.mem->ptr + c->offset, c->data.mem->used - c->offset - 1);
+                }
+
+                if (packet->b->used >= sizeof(*header) + 1) break;
+        }
+
+        if ((packet->b->used == 0) ||
+            (packet->b->used - 1 < sizeof(FCGI_Header))) {
+                /* no header */
+                buffer_free(packet->b);
+
+                log_error_write(srv, __FILE__, __LINE__, "s", "FastCGI: header to small");
+                return -1;
+        }
+
+        /* we have at least a header, now check how much me have to fetch */ 
+        header = (FCGI_Header *)(packet->b->ptr);
+                        
+        packet->len = (header->contentLengthB0 | (header->contentLengthB1 << 8)) + header->paddingLength;
+        packet->request_id = (header->requestIdB0 | (header->requestIdB1 << 8));
+        packet->type = header->type;
+        packet->padding = header->paddingLength;
+
+        /* the first bytes in packet->b are the header */
+        offset = sizeof(*header);
+
+        log_error_write(srv, __FILE__, __LINE__, "sddd", "FastCGI: got header:", packet->len, packet->request_id, packet->type);
+
+        /* ->b should only be the content */
+        buffer_reset(packet->b);
+
+        if (packet->len) {
+                /* copy the content */
+                for (; c && (packet->b->used < packet->len + 1); c = c->next) {
+                        toread = c->data.mem->used - c->offset - offset - 1 > packet->len ? packet->len : c->data.mem->used - c->offset - offset - 1;
+                        
+                        log_error_write(srv, __FILE__, __LINE__, "sdd", "FastCGI: reading content:", packet->b->used, toread);
+
+                        buffer_append_string_len(packet->b, c->data.mem->ptr + c->offset + offset, toread);
+                        offset = 0;
+                }
+
+                if (packet->b->used < packet->len + 1) {
+                        /* we didn't got the full packet */
+                        log_error_write(srv, __FILE__, __LINE__, "sdd", "FastCGI: not the full packet", packet->b->used, packet->len);
+
+                        buffer_free(packet->b);
+                        return -1;
+                }
+
+                packet->b->used -= packet->padding;
+                packet->b->ptr[packet->b->used - 1] = '\0';
+        }
+
+        /* tag the chunks as read */
+        toread = packet->len + sizeof(FCGI_Header);
+        for (c = hctx->rb->first; c && toread; c = c->next) {
+                if (c->data.mem->used - c->offset - 1 <= toread) {
+                        /* we read this whole buffer, move it to unused */
+                        toread -= c->data.mem->used - c->offset - 1;
+                        c->offset = c->data.mem->used - 1; /* everthing has been written */
+                } else {
+                        c->offset += toread;
+                        toread = 0;
+                }
+        }
+
+        chunkqueue_remove_finished_chunks(hctx->rb);
+        
+        return 0;
+}
 
 static int fcgi_demux_response(server *srv, handler_ctx *hctx) {
-        ssize_t len;
         int fin = 0;
-        int b;
+        int toread;
         ssize_t r;
         
@@ -1956 +2040 @@
          * check how much we have to read 
          */
-        if (ioctl(hctx->fd, FIONREAD, &b)) {
+        if (ioctl(hctx->fd, FIONREAD, &toread)) {
                 log_error_write(srv, __FILE__, __LINE__, "sd", 
                                 "unexpected end-of-file (perhaps the fastcgi process died):",
@@ -1964 +2048 @@
         
         /* init read-buffer */
-        if (hctx->rb == NULL) {
-                hctx->rb = calloc(1, sizeof(*hctx->rb));
-        }
-        
-        if (b > 0) {
-                if (hctx->rb->size == 0) {
-                        hctx->rb->size = b;
-                        hctx->rb->ptr = malloc(hctx->rb->size * sizeof(*hctx->rb->ptr));
-                } else if (hctx->rb->size < hctx->rb->used + b) {
-                        hctx->rb->size += b;
-                        hctx->rb->ptr = realloc(hctx->rb->ptr, hctx->rb->size * sizeof(*hctx->rb->ptr));
-                }
-                
+        
+        if (toread > 0) {
+                buffer *b;
+
+                b = chunkqueue_get_append_buffer(hctx->rb);
+                buffer_prepare_copy(b, toread + 1);
+
                 /* append to read-buffer */
-                if (-1 == (r = read(hctx->fd, hctx->rb->ptr + hctx->rb->used, b))) {
+                if (-1 == (r = read(hctx->fd, b->ptr, toread))) {
                         log_error_write(srv, __FILE__, __LINE__, "sds", 
                                         "unexpected end-of-file (perhaps the fastcgi process died):",
@@ -1987 +2065 @@
                 /* this should be catched by the b > 0 above */
                 assert(r);
-                
-                hctx->rb->used += r;
+
+                b->used = r + 1; /* one extra for the fake \0 */
+                b->ptr[b->used - 1] = '\0';
         } else {
                 log_error_write(srv, __FILE__, __LINE__, "ssdsdsd", 
@@ -1998 +2077 @@
                 return -1;
         }
-        
-        /* parse all fcgi packets 
-         * 
-         *   start: hctx->rb->ptr 
-         *   end  : hctx->rb->ptr + hctx->rb->used
-         * 
-         */
+
+        /*
+         * parse the fastcgi packets and forward the content to the write-queue
+         *
+         */     
         while (fin == 0) {
-                size_t request_id;
-                
-                if (hctx->response_len == 0) {
-                        FCGI_Header *header;
-                        
-                        if (hctx->rb->used - hctx->rb->offset < sizeof(*header)) {
-                                /* didn't get the full header packet (most often 0),
-                                 * but didn't recieved the final packet either
+                fastcgi_response_packet packet;
+
+                /* check if we have at least one packet */
+                if (0 != fastcgi_get_packet(srv, hctx, &packet)) {
+                        /* no full packet */
+
+                        hctx->delayed = 1;
+
+                        break;
+                }
+
+                switch(packet.type) {
+                case FCGI_STDOUT:
+                        if (packet.len == 0) break;
+
+                        /* is the header already finished */
+                        if (0 == con->file_started) {
+                                char *c;
+                                size_t blen;
+                                        
+                                /* search for header terminator 
                                  * 
-                                 * we will come back later and finish everything
-                                 * 
+                                 * if we start with \r\n check if last packet terminated with \r\n
+                                 * if we start with \n check if last packet terminated with \n
+                                 * search for \r\n\r\n
+                                 * search for \n\n
                                  */
-                                
-                                hctx->delayed = 1;
-#if 0
-                                log_error_write(srv, __FILE__, __LINE__, "sddd", "didn't get the full header: ",
-                                                hctx->rb->used - hctx->rb->offset, sizeof(*header),
-                                                fcgi_fd
-                                                );
-#endif
-                                break;
-                        }
-#if 0
-                        fprintf(stderr, "fcgi-version: %02x\n", hctx->rb->ptr[hctx->rb->offset]);
-#endif
-                        
-                        header = (FCGI_Header *)(hctx->rb->ptr + hctx->rb->offset);
-                        hctx->rb->offset += sizeof(*header);
-                        
-                        len = (header->contentLengthB0 | (header->contentLengthB1 << 8)) + header->paddingLength;
-                        request_id = (header->requestIdB0 | (header->requestIdB1 << 8));
-
-                        hctx->response_len = len;
-                        hctx->response_request_id = request_id;
-                        hctx->response_type = header->type;
-                        hctx->response_padding = header->paddingLength;
-                        
-#if 0
-                        log_error_write(srv, __FILE__, __LINE__, "sddd", "offset: ",
-                                        fcgi_fd, hctx->rb->offset, header->type
-                                        );
-#endif
-                        
-                } else {
-                        len = hctx->response_len;
-                }
-                
-                if (hctx->rb->used - hctx->rb->offset < hctx->response_len) {
-                        /* we are not finished yet */
-                        break;
-                }
-                
-                hctx->response->ptr = hctx->rb->ptr + hctx->rb->offset;
-                hctx->rb->offset += hctx->response_len;
-#if 0
-                log_error_write(srv, __FILE__, __LINE__, "sdd", "offset: ",
-                                fcgi_fd, hctx->rb->offset
-                                );
-#endif
-                
-                /* remove padding */
-#if 0
-                hctx->response->ptr[hctx->response_len - hctx->response_padding] = '\0';
-#endif
-                hctx->response->used = hctx->response_len - hctx->response_padding + 1;
-                
-                /* mark the fast-cgi packet as finished */
-                hctx->response_len = 0;
-                
-                switch(hctx->response_type) {
-                case FCGI_STDOUT:
-                        if (len) {
-#if 0
-                                log_error_write(srv, __FILE__, __LINE__, "sdb", "len", len, hctx->response);
-#endif
-                                
-                                if (0 == con->got_response) {
-                                        con->got_response = 1;
-                                        buffer_prepare_copy(hctx->response_header, 128);
+
+                                if (hctx->response_header->used == 0) {
+                                        buffer_copy_string_buffer(hctx->response_header, packet.b);
+                                } else {
+                                        buffer_append_string_buffer(hctx->response_header, packet.b);
                                 }
-                                
-                                if (0 == con->file_started) {
-                                        char *c;
-                                        size_t hlen;
-                                        
-                                        /* search for header terminator 
-                                         * 
-                                         * if we start with \r\n check if last packet terminated with \r\n
-                                         * if we start with \n check if last packet terminated with \n
-                                         * search for \r\n\r\n
-                                         * search for \n\n
-                                         */
-                                        
-                                        if (hctx->response->used > 2 &&
-                                            hctx->response->ptr[0] == '\r' &&
-                                            hctx->response->ptr[1] == '\n' &&
-                                            hctx->response_header->used > 3 &&
-                                            hctx->response_header->ptr[hctx->response_header->used - 3] == '\r' &&
-                                            hctx->response_header->ptr[hctx->response_header->used - 2] == '\n') {
-                                                hlen = 2;
-                                                c = hctx->response->ptr + 2;
-                                        } else if (hctx->response->used > 2 &&
-                                                   hctx->response->ptr[0] == '\n' &&
-                                                   hctx->response_header->used > 2 &&
-                                                   hctx->response_header->ptr[hctx->response_header->used - 2] == '\n') {
-                                                hlen = 1;
-                                                c = hctx->response->ptr + 1;
-                                        } else if (NULL != (c = buffer_search_string_len(hctx->response, "\r\n\r\n", 4))) {
-                                                c += 4;
-                                                hlen = c - hctx->response->ptr;
+
+                                if (NULL != (c = buffer_search_string_len(hctx->response_header, CONST_STR_LEN("\r\n\r\n")))) {
+                                        blen = hctx->response_header->used - (c - hctx->response_header->ptr) - 4;
+                                        hctx->response_header->used = c - hctx->response_header->ptr;
+                                        c += 4; /* point the the start of the response */
+                                } else if (NULL != (c = buffer_search_string_len(hctx->response_header, CONST_STR_LEN("\n\n")))) {
+                                        blen = hctx->response_header->used - (c - hctx->response_header->ptr) - 2;
+                                        hctx->response_header->used = c - hctx->response_header->ptr;
+                                        c += 2; /* point the the start of the response */
+                                } else {
+                                        /* no luck, no header found */
+                                        break;
+                                }
+
+                                /* parse the response header */
+                                fcgi_response_parse(srv, con, p, hctx->response_header);
                                                 
-                                        } else if (NULL != (c = buffer_search_string_len(hctx->response, "\n\n", 2))) {
-                                                c += 2;
-                                                hlen = c - hctx->response->ptr;
-                                        }
-                                        
-                                        if (c != NULL) {
-                                                size_t blen = hctx->response->used - hlen - 1;
-                                                /* found */
+                                if (host->mode != FCGI_AUTHORIZER ||
+                                    !(con->http_status == 0 ||
+                                      con->http_status == 200)) {
+                                                        
+                                        con->file_started = 1;
                                                 
-                                                buffer_append_string_len(hctx->response_header, hctx->response->ptr, hlen);
-#if 0
-                                                log_error_write(srv, __FILE__, __LINE__, "ss", "Header:", hctx->response_header->ptr);
-#endif
-                                                /* parse the response header */
-                                                fcgi_response_parse(srv, con, p, hctx->response_header);
-                                                
-                                                if (host->mode != FCGI_AUTHORIZER ||
-                                                    !(con->http_status == 0 ||
-                                                      con->http_status == 200)) {
-                                                        
-                                                        con->file_started = 1;
-                                                
-                                                        if (blen) {
-                                                                /* enable chunked-transfer-encoding */
-                                                                if (con->request.http_version == HTTP_VERSION_1_1 &&
-                                                                    !(con->parsed_response & HTTP_CONTENT_LENGTH)) {
-                                                                        con->response.transfer_encoding = HTTP_TRANSFER_ENCODING_CHUNKED;
-                                                                }
-
-