January 16, 2012 at 10:38 pm
· Filed under C10K, Life, nginx
Developing a Web Application Firewall module for Nginx is not an easy task. The lack of input body filters makes it harder. Nginx is an outstanding web server, but it is not perfect. Actually, nothing is perfect.
So we added the input body filter mechanism to our own Nginx distribution, which is named Tengine. By taking advantage of this mechanism, processing the request body is not that complicated anymore (In standard Nginx, request body may be buffered to disk file and you have to deal with up to two buffers)
Here I have an example to demonstrate how to write an input body filter. It is a simple module to fight hash collision DoS attacks.
/*
* Copyright (C) Joshua Zhu, http://www.zhuzhaoyuan.com
*/
#include <ngx_config.h>
#include <ngx_core.h>
#include <ngx_http.h>
typedef struct {
ngx_flag_t enable;
ngx_uint_t max_post_params;
} ngx_http_anti_hashdos_loc_conf_t;
typedef struct {
ngx_uint_t post_params;
} ngx_http_anti_hashdos_ctx_t;
static ngx_int_t ngx_http_anti_hashdos_input_body_filter(ngx_http_request_t *r,
ngx_buf_t *buf);
static void *ngx_http_anti_hashdos_create_loc_conf(ngx_conf_t *cf);
static char *ngx_http_anti_hashdos_merge_loc_conf(ngx_conf_t *cf, void *parent,
void *child);
static ngx_int_t ngx_http_anti_hashdos_init(ngx_conf_t *cf);
static ngx_command_t ngx_http_anti_hashdos_filter_commands[] = {
{ ngx_string("anti_hashdos"),
NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_FLAG,
ngx_conf_set_flag_slot,
NGX_HTTP_LOC_CONF_OFFSET,
offsetof(ngx_http_anti_hashdos_loc_conf_t, enable),
NULL },
{ ngx_string("anti_hashdos_max_post_params"),
NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1,
ngx_conf_set_num_slot,
NGX_HTTP_LOC_CONF_OFFSET,
offsetof(ngx_http_anti_hashdos_loc_conf_t, max_post_params),
NULL },
ngx_null_command
};
static ngx_http_module_t ngx_http_anti_hashdos_filter_module_ctx = {
NULL, /* preconfiguration */
ngx_http_anti_hashdos_init, /* postconfiguration */
NULL, /* create main configuration */
NULL, /* init main configuration */
NULL, /* create server configuration */
NULL, /* merge server configuration */
ngx_http_anti_hashdos_create_loc_conf,/* create location configuration */
ngx_http_anti_hashdos_merge_loc_conf /* merge location configuration */
};
ngx_module_t ngx_http_anti_hashdos_filter_module = {
NGX_MODULE_V1,
&ngx_http_anti_hashdos_filter_module_ctx, /* module context */
ngx_http_anti_hashdos_filter_commands,/* module directives */
NGX_HTTP_MODULE, /* module type */
NULL, /* init master */
NULL, /* init module */
NULL, /* init process */
NULL, /* init thread */
NULL, /* exit thread */
NULL, /* exit process */
NULL, /* exit master */
NGX_MODULE_V1_PADDING
};
static ngx_http_input_body_filter_pt ngx_http_next_input_body_filter;
static ngx_int_t
ngx_http_anti_hashdos_input_body_filter(ngx_http_request_t *r,
ngx_buf_t *buf)
{
u_char *p;
ngx_http_anti_hashdos_ctx_t *ctx;
ngx_http_anti_hashdos_loc_conf_t *ahlf;
ahlf = ngx_http_get_module_loc_conf(r, ngx_http_anti_hashdos_filter_module);
if (!ahlf->enable) {
return ngx_http_next_input_body_filter(r, buf);
}
ctx = ngx_http_get_module_ctx(r, ngx_http_anti_hashdos_filter_module);
if (ctx == NULL) {
ctx = ngx_pcalloc(r->pool, sizeof(ngx_http_anti_hashdos_ctx_t));
if (ctx == NULL) {
return NGX_HTTP_INTERNAL_SERVER_ERROR;
}
ctx->post_params = 1;
}
for (p = buf->pos; p < buf->last; p++) {
if (*p == '&') {
ctx->post_params++;
}
}
if (ctx->post_params > ahlf->max_post_params) {
ngx_log_error(NGX_LOG_ERR, r->connection->log, 0,
"anti hashdos: \"%V\" blocked, too many post params: %d",
&r->connection->addr_text,
ctx->post_params);
return NGX_HTTP_BAD_REQUEST;
}
return ngx_http_next_input_body_filter(r, buf);
}
static void *
ngx_http_anti_hashdos_create_loc_conf(ngx_conf_t *cf)
{
ngx_http_anti_hashdos_loc_conf_t *conf;
conf = ngx_pcalloc(cf->pool, sizeof(ngx_http_anti_hashdos_loc_conf_t));
if (conf == NULL) {
return NULL;
}
conf->enable = NGX_CONF_UNSET;
conf->max_post_params = NGX_CONF_UNSET_UINT;
return conf;
}
static char *
ngx_http_anti_hashdos_merge_loc_conf(ngx_conf_t *cf, void *parent, void *child)
{
ngx_http_anti_hashdos_loc_conf_t *prev = parent;
ngx_http_anti_hashdos_loc_conf_t *conf = child;
ngx_conf_merge_value(conf->enable, prev->enable, 0);
ngx_conf_merge_uint_value(conf->max_post_params, prev->max_post_params,
120);
return NGX_CONF_OK;
}
static ngx_int_t
ngx_http_anti_hashdos_init(ngx_conf_t *cf)
{
ngx_http_next_input_body_filter = ngx_http_top_input_body_filter;
ngx_http_top_input_body_filter = ngx_http_anti_hashdos_input_body_filter;
return NGX_OK;
}The code looks quite straight forward, right? And it is similar to an output body filter, just a few steps:
1) Implement your own input body filter function. e.g.
static ngx_int_t
ngx_http_anti_hashdos_input_body_filter(ngx_http_request_t *r, ngx_buf_t *buf)
{
/* Do the input body filtering here */
}2) In your input body filter function, return an HTTP error code if something is wrong. Otherwise, call ngx_http_next_input_body_filter(r, buf) directly to pass the buf to the next input body filters.
3) Install your input body filter in the post_configuration hook function. Push your input body filter to the head of the input body filter chain. e.g.
static ngx_int_t
ngx_http_anti_hashdos_init(ngx_conf_t *cf)
{
ngx_http_next_input_body_filter = ngx_http_top_input_body_filter;
ngx_http_top_input_body_filter = ngx_http_anti_hashdos_input_body_filter;
return NGX_OK;
}NOTE: This is just a demonstration to show how to write input body filters. If you want to fight hash collision DoS attacks completely, you have to write more code and various POST content types should be processed.
Download the code here:
http://www.zhuzhaoyuan.com/download/tengine/anti_hashdos.tar.gz
Permalink
December 2, 2011 at 7:31 pm
· Filed under C10K, nginx
We’re glad to announce that Tengine, our home-baked Nginx at Taobao now becomes an open source project.
Taobao is the largest e-commerce website in Asia and ranked #12 on Alexa’s top global sites list. Our website serves billions of pageviews per day. For busy website as us, Nginx is obviously the best choice. Thanks to Nginx’s high performance, small footprint and flexibility, we have done more with less.
We first learned the Nginx internals by using it as a traditional web server and developing dozens of modules. Then from June of this year we started hacking the Nginx core to expand its capabilities. As some of the features we have developed may also benefit other Nginx users and websites, so why not open source them? We do not want to be just open source software users, but also open source contributors. That’s why the Tengine open source project came out.
Tengine is based on the latest stable version of Nginx (Nginx-1.0.10). There are a few features and bug fixes you may be interested in Tengine:
- Logging enhancement. It supports syslog (local and remote) and pipe logging. You can also do log sampling, i.e. not all requests have to be written.
- Protects the server when the system load and memory use goes high.
- Combines multiple CSS or JavasScript requests into one request to reduce the downloading time.
- Sets the worker process number and CPU affinities automatically. Setting Nginx’s worker_cpu_affinity is not a pain any more.
- Enhanced limit_req module with whitelist support and more limit_req directives in one location.
- More operations engineer friendly server information, so host can be located easily when error happens.
- More command lines support. You can list all modules compiled in and the directives supported, even the content of configuration file itself.
- Set expiration for files according to specific content type.
- Error pages can be set back to ‘default’.
Basically, Tengine can be considered as a better or superset of Nginx. You can download the tar ball here:
http://tengine.taobao.org/download/tengine-1.2.0.tar.gz
We want to say thank you to the Nginx team, especially to Igor. Thank you very much for your great work! We would love to donate the patches against the Nginx-1.1 branch later if you think the patches are okay.
Frankly, I’m not sure whether the features in Tengine right now can impress you guys or not. It’s the first step we moving towards open source after all. We have built a team working on Tengine and have quite a long to-do list. I promise you more enhancements are coming out.
Permalink
September 22, 2009 at 4:14 pm
· Filed under C10K, Software, nginx
Last Saturday I gave the talk “Nginx Internals” in Guangzhou. Here are the presentation slides and the video of the talk.
Nginx Internals Video part 1 (in Chinese):
Nginx Internals Video part 2 (in Chinese):
UPDATE: This part is lost, saddly, and I have no backup 
Nginx Internals Video part 3 (in Chinese):
Permalink
August 31, 2009 at 11:37 am
· Filed under C10K, Software, nginx
nginx map (click to view large image)
I’m going to give a free talk on nginx’s internals next month (September 19), in Guangzhou, China.
I’ve been reading the source code of nginx for a few days. Digging into this charming code is really a pleasant experience, though at first glance it appeared a little bit difficult to understand. Nginx becomes more and more popular, but unfortunately there is not enough documentation on its architecture and implementation. Now that I have spent a considerable amount of time reading the source code and have gained some knowledge, why not share it with those who want to know things under the hood?
So, if you are interested in this talk and you can be in Guangzhou that day, feel free to join in. Please comment on this post or drop me an email to let me know which parts you are interested in (see the mind map above, draft version though).
There might be a thousand Hamlets in a thousand people’s eyes. Note that I’m not Igor, and the only way I try to understand the nuts and bolts is by reverse engineering it, hence I can’t guarantee you no mistakes or misunderstandings in my talk. And frankly, it is not a trivial topic after all, not only because of the size of nginx’s code base, but also its elaborate design.
The speech will be in Chinese while slides will be in English. Specifics of time and location are coming soon. Stay tuned.
Update:
Time: 14:30-17:30, September 19, 2009
Location: Netease Building Tower E, Guangzhou Information Port #16 Keyun RD. Tianhe District, Guangzhou
Registration: http://blog.laiyonghao.com/2009/09/programming-tech-party/370
Permalink
August 13, 2009 at 3:53 pm
· Filed under Programming, nginx
Some of my friends think nginx modules are very difficult to write. Sure it’s not so easy but it’s not that hard either. The only problem is that the documentation is not enough. But don’t let this scare yourself away, the situation is improving.
So I write a hello world nginx module here. It’s pretty short and with enough comments. If you’re a new nginx module developer, feel free to take it as an example and replace hello with whatever your module name is, then start your happy nginx module hacking journey. And I’ll write more topics on nginx soon. Look for it! 
ngx_http_hello_module.c:
/*
* Copyright (C) Joshua Zhu, http://www.zhuzhaoyuan.com
*/
#include <ngx_config.h>
#include <ngx_core.h>
#include <ngx_http.h>
static char *ngx_http_hello(ngx_conf_t *cf, ngx_command_t *cmd,
void *conf);
static ngx_command_t ngx_http_hello_commands[] = {
{ ngx_string("hello"),
NGX_HTTP_LOC_CONF|NGX_CONF_NOARGS,
ngx_http_hello,
0,
0,
NULL },
ngx_null_command
};
static u_char ngx_hello_string[] = "Hello, world!";
static ngx_http_module_t ngx_http_hello_module_ctx = {
NULL, /* preconfiguration */
NULL, /* postconfiguration */
NULL, /* create main configuration */
NULL, /* init main configuration */
NULL, /* create server configuration */
NULL, /* merge server configuration */
NULL, /* create location configuration */
NULL /* merge location configuration */
};
ngx_module_t ngx_http_hello_module = {
NGX_MODULE_V1,
&ngx_http_hello_module_ctx, /* module context */
ngx_http_hello_commands, /* module directives */
NGX_HTTP_MODULE, /* module type */
NULL, /* init master */
NULL, /* init module */
NULL, /* init process */
NULL, /* init thread */
NULL, /* exit thread */
NULL, /* exit process */
NULL, /* exit master */
NGX_MODULE_V1_PADDING
};
static ngx_int_t
ngx_http_hello_handler(ngx_http_request_t *r)
{
ngx_int_t rc;
ngx_buf_t *b;
ngx_chain_t out;
/* we response to 'GET' and 'HEAD' requests only */
if (!(r->method & (NGX_HTTP_GET|NGX_HTTP_HEAD))) {
return NGX_HTTP_NOT_ALLOWED;
}
/* discard request body, since we don't need it here */
rc = ngx_http_discard_request_body(r);
if (rc != NGX_OK) {
return rc;
}
/* set the 'Content-type' header */
r->headers_out.content_type_len = sizeof("text/html") - 1;
r->headers_out.content_type.len = sizeof("text/html") - 1;
r->headers_out.content_type.data = (u_char *) "text/html";
/* send the header only, if the request type is http 'HEAD' */
if (r->method == NGX_HTTP_HEAD) {
r->headers_out.status = NGX_HTTP_OK;
r->headers_out.content_length_n = sizeof(ngx_hello_string) - 1;
return ngx_http_send_header(r);
}
/* allocate a buffer for your response body */
b = ngx_pcalloc(r->pool, sizeof(ngx_buf_t));
if (b == NULL) {
return NGX_HTTP_INTERNAL_SERVER_ERROR;
}
/* attach this buffer to the buffer chain */
out.buf = b;
out.next = NULL;
/* adjust the pointers of the buffer */
b->pos = ngx_hello_string;
b->last = ngx_hello_string + sizeof(ngx_hello_string) - 1;
b->memory = 1; /* this buffer is in memory */
b->last_buf = 1; /* this is the last buffer in the buffer chain */
/* set the status line */
r->headers_out.status = NGX_HTTP_OK;
r->headers_out.content_length_n = sizeof(ngx_hello_string) - 1;
/* send the headers of your response */
rc = ngx_http_send_header(r);
if (rc == NGX_ERROR || rc > NGX_OK || r->header_only) {
return rc;
}
/* send the buffer chain of your response */
return ngx_http_output_filter(r, &out);
}
static char *
ngx_http_hello(ngx_conf_t *cf, ngx_command_t *cmd, void *conf)
{
ngx_http_core_loc_conf_t *clcf;
clcf = ngx_http_conf_get_module_loc_conf(cf, ngx_http_core_module);
clcf->handler = ngx_http_hello_handler; /* handler to process the 'hello' directive */
return NGX_CONF_OK;
}config:
ngx_addon_name=ngx_http_hello_module
HTTP_MODULES="$HTTP_MODULES ngx_http_hello_module"
NGX_ADDON_SRCS="$NGX_ADDON_SRCS $ngx_addon_dir/ngx_http_hello_module.c"
hello.conf:
# hello
worker_processes 1;
events {
worker_connections 1024;
}
http {
server {
listen 8080;
server_name localhost;
location / {
hello;
}
}
}If you want more detailed information, please consult Evan Miller’s Guide To Nginx Module Development.
Permalink
