initial commit master
initial commit

file:b/.jshintrc (new)
--- /dev/null
+++ b/.jshintrc
@@ -1,1 +1,35 @@
+{
+  "node": true,
+  "browser": false,
+  "esnext": true,
+  "bitwise": true,
+  "camelcase": false,
+  "curly": true,
+  "eqeqeq": true,
+  "immed": true,
+  "indent": 2,
+  "latedef": true,
+  "newcap": true,
+  "noarg": true,
+  "quotmark": "single",
+  "regexp": true,
+  "undef": true,
+  "unused": true,
+  "strict": false,
+  "trailing": true,
+  "smarttabs": true,
+  "globals": {
+    "afterEach": true,
+    "backendCalled": true,
+    "before": true,
+    "beforeEach": true,
+    "describe": true,
+    "Factory": true,
+    "gatekeeper": true,
+    "it": true,
+    "mongoose": true,
+    "shared": true,
+    "request": true
+  }
+}
 

file:b/.npmignore (new)
--- /dev/null
+++ b/.npmignore
@@ -1,1 +1,4 @@
+/node_modules
+/cache
+/test/cache
 

file:b/Gruntfile.js (new)
--- /dev/null
+++ b/Gruntfile.js
@@ -1,1 +1,29 @@
+module.exports = function(grunt) {
+    grunt.loadNpmTasks('grunt-contrib-jshint');
+    grunt.loadNpmTasks('grunt-mocha-test');
 
+    grunt.initConfig({
+        jshint: {
+            options: {
+                jshintrc: '.jshintrc'
+            },
+            all: [
+                'Gruntfile.js',
+                'index.js',
+                'lib/**/*.js',
+                'test/**/*.js',
+            ],
+        },
+        mochaTest: {
+            test: {
+                options: {
+                    reporter: 'spec',
+                },
+                src: ['test/**/*.js']
+            },
+        },
+    });
+
+    grunt.registerTask('default', ['jshint', 'mochaTest']);
+};
+

file:b/README.md (new)
--- /dev/null
+++ b/README.md
@@ -1,1 +1,24 @@
+# http-memcache-proxy
 
+A non-compliant HTTP caching proxy.
+
+The behavior of this proxy is:
+
+* Non-cached content is transparently proxied (using [node-http-proxy]).
+  * Successful GET or HEAD requests will be cached for subsequent use.
+* Cached content will be delivered from memcache.
+
+## Usage
+
+```js
+var httpMemCacheProxy = require('./http-memcache-proxy');
+
+httpMemCacheProxy.createServer({
+  changeOrigin: true,
+  target: {
+    host: 'drupal-demo.dev.webstyler.ro',
+    port: 80,
+  },
+  memcache: {'127.0.0.1:11211': 1},
+}).listen(8000);
+```

file:b/example.js (new)
--- /dev/null
+++ b/example.js
@@ -1,1 +1,11 @@
+var httpMemcacheProxy = require('./lib/http-memcache-proxy');
 
+httpMemcacheProxy.createServer({
+  changeOrigin: true,
+  target: {
+    host: 'git.razvi.ro',
+    port: 80,
+  },
+  memcache: {'127.0.0.1:11211': 1},
+}).listen(2380);
+

--- /dev/null
+++ b/lib/http-memcache-proxy.js
@@ -1,1 +1,146 @@
+var _ = require('lodash'),
+    async = require('async'),
+    crypto = require('crypto'),
+    httpMocks = require('node-mocks-http'),
+    httpProxy = require('http-proxy'),
+    logger = require('./logger'),
+    path = require('path'),
+    memcached = require('memcached');
 
+var cacheBase = process.env.PROXY_CACHE_DIR || path.resolve(process.cwd(), './cache/');
+
+var Cache = function() {
+    this.initialize.apply(this, arguments);
+};
+
+_.extend(Cache.prototype, {
+    initialize: function(request, response, proxy, options) {
+        this.request = request;
+        this.response = response;
+        this.proxy = proxy;
+        this.options = options;
+        this.buffer = httpProxy.buffer(request);
+        this.memcache = new memcached(options.memcache, {});
+
+        this.requestLogId = request.method + ' ' + request.url;
+
+        var cacheKey = [
+            request.method,
+            request.headers.host,
+            request.url,
+            request.headers.authorization,
+        ].join('');
+
+        this.cachePath = crypto.createHash('sha256').update(cacheKey).digest('hex');
+
+        // TODO: Make this more configurable.
+        this.cacheDecider = function() {
+            return true;
+        };
+
+        this.metaCachePath = path.join(cacheBase, this.cachePath + '.meta');
+        this.bodyCachePath = path.join(cacheBase, this.cachePath + '.body');
+
+        console.log(this.requestLogId + ' - Request received');
+        this.checkCache(this.handleCheckCache.bind(this));
+    },
+
+    checkCache: function(callback) {
+        this.memcache.get(this.bodyCachePath, function(error, response){
+            if (response != null) {
+                console.log(this.requestLogId + ' - Delivering cached response');
+                response = JSON.parse(response);
+                this.response.writeHead(response.metadata.statusCode, response.metadata.headers);
+                this.response.write(response.html);
+                this.response.end();
+                //callback(true);
+            } else {
+                callback(false);
+            }
+        }.bind(this));
+    },
+
+    writeCacheBody: function(upstreamResponse, body, callback) {
+        var data = {};
+        delete(upstreamResponse.headers['content-length']);
+        //delete(upstreamResponse.headers['content-encoding']);
+        data.metadata = {
+            request: {
+                method: this.request.method,
+                url: this.request.url,
+                headers: this.request.headers,
+            },
+            statusCode: upstreamResponse.statusCode,
+            headers: upstreamResponse.headers,
+        };
+        data.html = body;
+        data = JSON.stringify(data, null, null);
+        this.memcache.set(this.bodyCachePath, data, 15, function(err){
+        });
+    },
+
+    handleCheckCache: function(cached) {
+        if(cached) {
+            this.respondFromCache(function(error) {
+                if(error) {
+                    this.proxyAndCache(this.response);
+                } else {
+                    this.refreshCache();
+                }
+            }.bind(this));
+        } else {
+            this.proxyAndCache(this.response);
+        }
+    },
+
+    refreshCache: function() {
+        console.log(this.requestLogId + ' - Asynchronously refreshing response');
+
+        var dummyResponse = httpMocks.createResponse();
+        this.proxyAndCache(dummyResponse);
+    },
+
+    proxyAndCache: function(response) {
+        console.log(this.requestLogId + ' - Proxying request');
+
+        this.proxy.proxyRequest(this.request, response, _.extend({}, this.options, {
+            buffer: this.buffer,
+        }));
+    },
+
+    respondFromCache: function(callback) {
+        console.log(this.requestLogId + ' - Delivering cached response');
+        callback(true);
+    },
+
+    writeCache: function(upstreamResponse, body) {
+        if(this.request.method === 'GET' || this.request.method === 'HEAD') {
+            if(upstreamResponse.statusCode >= 200 && upstreamResponse.statusCode < 400) {
+                async.parallel([
+                    this.writeCacheBody.bind(this, upstreamResponse, body)
+                ]);
+            }
+        }
+    }
+});
+
+exports.createServer = function(options) {
+    var server = httpProxy.createServer(function (request, response, proxy) {
+        delete(request.headers['accept-encoding']);
+        request.cacher = new Cache(request, response, proxy, options);
+        //response.writeHead(200, { 'Content-Type': 'text/plain' });
+        //response.write('request successfully proxied to: ' + request.url + '\n' + JSON.stringify(request.headers, true, 2));
+        //response.end();
+    });
+
+    server.proxy.on('proxyResponse', function(request, response, upstreamResponse) {
+        var body = '', obj
+        upstreamResponse.on('data', function (c) { body += c})
+        upstreamResponse.on('end', function (c) {
+            request.cacher.writeCache(upstreamResponse, body);
+        });
+    });
+
+    return server;
+};
+

file:b/lib/logger.js (new)
--- /dev/null
+++ b/lib/logger.js
@@ -1,1 +1,27 @@
+var path = require('path'),
+    winston = require('winston');
 
+var logPath = process.env.PROXY_LOG || path.join(process.cwd(), 'proxy.log');
+
+var logger = new (winston.Logger)({
+  transports: [
+    new (winston.transports.File)({
+      filename: logPath,
+      json: false,
+    })
+  ],
+
+  levels: {
+    debug: 0,
+    info: 1,
+    notice: 2,
+    warning: 3,
+    error: 4,
+    crit: 5,
+    alert: 6,
+    emerg: 7
+  },
+});
+
+module.exports = logger;
+

file:b/package.json (new)
--- /dev/null
+++ b/package.json
@@ -1,1 +1,36 @@
+{
+  "name": "http-memcache-proxy",
+  "description": "A non-compliant HTTP caching proxy",
+  "version": "0.0.1",
+  "engines": ">= 0.8.0",
+  "author": "Razvan Stanga <git@razvi.ro>",
+  "main": "./lib/http-memcache-proxy",
+  "scripts": {
+    "test": "grunt"
+  },
+  "dependencies": {
+    "async": "~0.2.9",
+    "http-proxy": "~0.10.3",
+    "lodash": "~2.0.0",
+    "node-mocks-http": "~1.0.1",
+    "winston": "~0.7.2",
+    "memcached": "~2.2.1"
+  },
+  "devDependencies": {
+    "chai": "~1.8.0",
+    "express": "~3.4.0",
+    "grunt": "~0.4.1",
+    "grunt-cli": "~0.1.9",
+    "grunt-contrib-jshint": "~0.6.4",
+    "grunt-mocha-test": "~0.7.0",
+    "mocha": "~1.13.0",
+    "request": "~2.27.0",
+    "rimraf": "~2.2.2"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/razvanstanga/http-memcache-proxy.git"
+  },
+  "license": "MIT"
+}
 

file:b/test/proxy.js (new)
--- /dev/null
+++ b/test/proxy.js
@@ -1,1 +1,128 @@
+require('./test_helper');
 
+var async = require('async'),
+    crypto = require('crypto'),
+    fs = require('fs.extra'),
+    path = require('path'),
+    request = require('request'),
+    rimraf = require('rimraf');
+
+describe('proxying', function() {
+  beforeEach(function(done) {
+    var cacheDir = process.env.PROXY_CACHE_DIR;
+    var cacheTempDir = path.join(process.env.PROXY_CACHE_DIR, 'tmp');
+    rimraf(process.env.PROXY_CACHE_DIR, function() {
+      fs.mkdirRecursiveSync(cacheDir);
+      fs.mkdirRecursiveSync(cacheTempDir);
+      done();
+    });
+  });
+
+  it('proxies to the backend if nothing is cached', function(done) {
+    var randomInput = Math.random().toString();
+    request.get('http://localhost:9333/echo?input=' + randomInput, function(error, response, body) {
+      body.should.eql(randomInput);
+      done();
+    });
+  });
+
+  it('writes the cache files after successfully proxying a request', function(done) {
+    var randomInput = Math.random().toString();
+    var url = '/echo?input=' + randomInput;
+
+    var cacheKey = ['GET', 'localhost:9333', url].join('');
+    var cacheBase = crypto.createHash('sha256').update(cacheKey).digest('hex');
+
+    request.get('http://localhost:9333' + url, function() {
+      setTimeout(function() {
+        var cachedMeta = fs.readFileSync(path.join(process.env.PROXY_CACHE_DIR, cacheBase + '.meta'));
+        var cachedBody = fs.readFileSync(path.join(process.env.PROXY_CACHE_DIR, cacheBase + '.body'));
+
+        JSON.parse(cachedMeta).request.url.should.eql(url);
+        cachedBody.toString().should.eql(randomInput);
+
+        done();
+      }, 10);
+    });
+  });
+
+  it('serves the previously cached response', function(done) {
+    request.get('http://localhost:9333/rand', function(error, response, body) {
+      var firstBody = body;
+
+      setTimeout(function() {
+        request.get('http://localhost:9333/rand', function(error, response, body) {
+          body.should.eql(firstBody);
+          done();
+        });
+      }, 10);
+    });
+  });
+
+  it('serves up the last cached response', function(done) {
+    request.get('http://localhost:9333/rand', function() {
+      request.get('http://localhost:9333/rand', function(error, response, body) {
+        var secondBody = body;
+
+        setTimeout(function() {
+          request.get('http://localhost:9333/rand', function(error, response, body) {
+            body.should.eql(secondBody);
+            done();
+          });
+        }, 10);
+      });
+    });
+  });
+
+  it('separates cached content by domain', function(done) {
+    request.get({ url: 'http://localhost:9333/rand', headers: { 'Host': 'example.com' } }, function(error, response, body) {
+      var firstBody = body;
+
+      setTimeout(function() {
+        request.get({ url: 'http://localhost:9333/rand', headers: { 'Host': 'example2.com' } }, function(error, response, body) {
+          body.should.not.eql(firstBody);
+          done();
+        });
+      }, 10);
+    });
+  });
+
+
+  it('caches concurrent requests to the appropriate file', function(done) {
+    // Fire off 20 concurrent requests and ensure that all the cached responses
+    // end up in the appropriate place.
+    async.times(20, function(index, next) {
+      var randomInput = Math.random().toString();
+      var url = '/echo_chunked?input=' + randomInput;
+
+      var cacheKey = ['GET', 'localhost:9333', url].join('');
+      var cacheBase = crypto.createHash('sha256').update(cacheKey).digest('hex');
+
+      request.get('http://localhost:9333' + url, function(error, response, body) {
+        next(null, {
+          url: url,
+          input: randomInput,
+          output: body,
+          cacheBase: cacheBase,
+        });
+      });
+    }, function(error, requests) {
+      setTimeout(function() {
+        for(var i = 0; i < requests.length; i++) {
+          var request = requests[i];
+          var cacheBase = request.cacheBase;
+
+          var cachedMeta = fs.readFileSync(path.join(process.env.PROXY_CACHE_DIR, cacheBase + '.meta'));
+          var cachedBody = fs.readFileSync(path.join(process.env.PROXY_CACHE_DIR, cacheBase + '.body'));
+
+          JSON.parse(cachedMeta).request.url.should.eql(request.url);
+          request.input.should.eql(request.output);
+          cachedBody.toString().should.eql(request.output);
+        }
+
+        done();
+      }, 50);
+    });
+  });
+});
+

--- /dev/null
+++ b/test/support/example_backend_app.js
@@ -1,1 +1,32 @@
+var async = require('async'),
+    express = require('express');
 
+var app = express();
+
+app.get('/echo', function(req, res) {
+  res.send(req.query.input);
+});
+
+app.get('/echo_chunked', function(req, res) {
+  var parts = req.query.input.split('');
+  async.eachSeries(parts, function(part, next) {
+    setTimeout(function() {
+      res.write(part);
+      next();
+    }, 10);
+  }, function() {
+    setTimeout(function() {
+      res.end('');
+    }, 10);
+  });
+});
+
+app.get('/rand', function(req, res) {
+  setTimeout(function() {
+    var rand = Math.random().toString();
+    res.send(rand);
+  }, 50);
+});
+
+app.listen(9444);
+

--- /dev/null
+++ b/test/test_helper.js
@@ -1,1 +1,17 @@
+var path = require('path');
+process.env.PROXY_CACHE_DIR = path.resolve(__dirname, './cache');
 
+var httpMemcacheProxy = require('../lib/http-memcache-proxy');
+
+require('./support/example_backend_app');
+
+httpMemcacheProxy.createServer({
+  changeOrigin: true,
+  target: {
+    host: 'localhost',
+    port: 9444,
+  },
+}).listen(9333);
+
+require('chai').should();
+

comments