From d2460adf36b14620fdd3a8a85ceddd00aa5a40d4 Mon Sep 17 00:00:00 2001 From: Inga Lovinde <52715130+inga-lovinde@users.noreply.github.com> Date: Fri, 22 Nov 2013 11:43:51 +0400 Subject: [PATCH] First working draft --- .gitignore => BuildServer/.gitignore | 3 + BuildServer/app.js | 48 ++++++++++++ BuildServer/lib/builder.js | 63 ++++++++++++++++ BuildServer/lib/git-loader.js | 95 ++++++++++++++++++++++++ BuildServer/lib/task-processor.js | 79 ++++++++++++++++++++ BuildServer/lib/tasks/echo.js | 21 ++++++ BuildServer/package.json | 15 ++++ BuildServer/public/stylesheets/style.css | 8 ++ BuildServer/routes/index.js | 12 +++ BuildServer/routes/manual.js | 19 +++++ BuildServer/routes/postreceive.js | 29 ++++++++ BuildServer/routes/status.js | 5 ++ BuildServer/views/index.jade | 5 ++ BuildServer/views/layout.jade | 7 ++ BuildServer/views/manual-done.jade | 9 +++ BuildServer/views/manual-done.jade.bak | 8 ++ BuildServer/views/manual.jade | 24 ++++++ BuildServer/views/status-image.jade | 8 ++ 18 files changed, 458 insertions(+) rename .gitignore => BuildServer/.gitignore (89%) create mode 100644 BuildServer/app.js create mode 100644 BuildServer/lib/builder.js create mode 100644 BuildServer/lib/git-loader.js create mode 100644 BuildServer/lib/task-processor.js create mode 100644 BuildServer/lib/tasks/echo.js create mode 100644 BuildServer/package.json create mode 100644 BuildServer/public/stylesheets/style.css create mode 100644 BuildServer/routes/index.js create mode 100644 BuildServer/routes/manual.js create mode 100644 BuildServer/routes/postreceive.js create mode 100644 BuildServer/routes/status.js create mode 100644 BuildServer/views/index.jade create mode 100644 BuildServer/views/layout.jade create mode 100644 BuildServer/views/manual-done.jade create mode 100644 BuildServer/views/manual-done.jade.bak create mode 100644 BuildServer/views/manual.jade create mode 100644 BuildServer/views/status-image.jade diff --git a/.gitignore b/BuildServer/.gitignore similarity index 89% rename from .gitignore rename to BuildServer/.gitignore index a72b52e..4fafb13 100644 --- a/.gitignore +++ b/BuildServer/.gitignore @@ -13,3 +13,6 @@ results npm-debug.log node_modules + +data +*.crt \ No newline at end of file diff --git a/BuildServer/app.js b/BuildServer/app.js new file mode 100644 index 0000000..e319430 --- /dev/null +++ b/BuildServer/app.js @@ -0,0 +1,48 @@ + +/** + * Module dependencies. + */ + +var https = require('https'); +var fs = require('fs'); + +https.globalAgent.options.ca = https.globalAgent.options.ca || []; +https.globalAgent.options.ca.push(fs.readFileSync("POS-CA.crt")); + +var express = require('express'); +var routes = require('./routes'); +var http = require('http'); +var path = require('path'); + +var app = express(); + +// all environments +app.set('port', process.env.PORT || 3000); +app.set('views', path.join(__dirname, 'views')); +app.set('view engine', 'jade'); +app.set('gitpath', 'data/git'); +app.set('tmpcodepath', 'data/code'); +app.set('releasepath', 'data/release'); +app.use(express.favicon()); +app.use(express.logger('dev')); +app.use(express.json()); +app.use(express.urlencoded()); +app.use(express.methodOverride()); +app.use(app.router); +app.use(express.static(path.join(__dirname, 'public'))); + +// development only +if ('development' == app.get('env')) { + app.use(express.errorHandler()); +} + +app.get('/', routes.index); +app.post('/github/postreceive', routes.postreceive); +app.get('/manual', routes.manual.get); +app.post('/manual', routes.manual.post); +app.get('/status', routes.status.image); +app.get('/status.svg', routes.status.image); + +http.createServer(app).listen(app.get('port'), function(){ + console.log('Express server listening on port ' + app.get('port')); +}); diff --git a/BuildServer/lib/builder.js b/BuildServer/lib/builder.js new file mode 100644 index 0000000..7fdc555 --- /dev/null +++ b/BuildServer/lib/builder.js @@ -0,0 +1,63 @@ +"use strict"; + +var fs = require('fs'); +var fse = require('fs-extra'); +var gitLoader = require('./git-loader'); +var processor = require('./task-processor'); + +var build = function (options, callback) { + var url = options.url, + owner = options.owner, + reponame = options.reponame, + rev = options.rev, + branch = options.branch, + local = options.app.get('gitpath') + "/" + owner + "/" + reponame + ".git", + tmp = options.app.get('tmpcodepath') + "/" + owner + "/" + reponame + "/" + branch + "/" + rev, + exported = tmp + "/code", + release = options.app.get('releasepath') + "/" + owner + "/" + reponame + "/" + branch + "/" + rev; + + fse.mkdirsSync(release); + + gitLoader({ + remote: url + ".git", + local: local, + branch: branch, + hash: rev, + exported: tmp + "/code", + }, function(err) { + if (err) { + console.log(err); + } + console.log("Done loading from git"); + fs.exists(exported + "/mbs.json", function (exists) { + if (!exists) { + return callback(null, "MBSNotFound"); + } + fs.readFile(exported + "/mbs.json", function (err, data) { + if (err) { + return callback(err, "MBSUnableToRead"); + } + + var task; + try { + task = JSON.parse(data); + } catch(err) { + console.log("Malformed data: " + data); + return callback(err, "MBSMalformed"); + } + + processor.processTask(task, function (err, result) { + if (err) { + return callback(err, result); + } + + fs.writeFile(release + "/report.json", JSON.stringify(result), function (err) { + return callback(err, result); + }); + }); + }); + }); + }); +} + +exports.build = build; diff --git a/BuildServer/lib/git-loader.js b/BuildServer/lib/git-loader.js new file mode 100644 index 0000000..296355f --- /dev/null +++ b/BuildServer/lib/git-loader.js @@ -0,0 +1,95 @@ +"use strict"; + +var git = require('git-node'), + async = require('async'), + fs = require('fs'), + fse = require('fs-extra'), + basename = require('path').basename, + mkdirs = function (path) { + /*jslint stupid: true */ + fse.mkdirsSync(path); + }, + removedirs = function (path) { + /*jslint stupid: true */ + fse.removeSync(path); + }; + +/* +options = { + "remote": "https://github.com/visionmedia/express.git", + "local": "D:\\data\\repositories\\visionmedia\\express.git\\", + "branch": "1.x", + "hash": "82e15cf321fccf3215068814d1ea1aeb3581ddb3", + "exported": "D:\\data\\exportedsource\\visionmedia\\express\\82e15cf321fccf3215068814d1ea1aeb3581ddb3\\", +} +*/ + +module.exports = function (options, globalCallback) { + var url = options.remote, + remote = git.remote(url), + path = options.local, + repo = git.repo(path), + exported = options.exported, + opts = { + want: options.branch, + /*onProgress: function (progress) { + process.stderr.write(progress); + }*/ + }, + done = function () { + globalCallback(); + }; + + mkdirs(path); + + //console.log("Cloning %s to %s", url, path); + + repo.fetch(remote, opts, function (err) { + if (err) { + return globalCallback(err); + } + + //console.log("Done fetching"); + + removedirs(exported); + mkdirs(exported); + + var q = async.queue(function (task, callback) { + //console.log("Going to write file " + task.path + " (" + task.buffer.length + " bytes)"); + fs.writeFile(exported + "/" + task.path, task.buffer, function (err, result) { + //console.log("Done writing file " + task.path); + callback(err, result); + }); + }, 10); + + repo.treeWalk(options.hash, function (err, tree) { + if (err) { + return globalCallback(err); + } + + var onEntry = function (err, entry) { + if (err) { + return globalCallback(err); + } + + if (!entry) { + if (q.length() === 0) { + process.nextTick(done); + } else { + q.drain = done; + } + return; + } + //console.log(" %s %s (%s)", entry.hash, entry.path, entry.type); + if (entry.type === "tree") { + mkdirs(exported + "/" + entry.path); + } else if (entry.type === "blob") { + q.push({path: entry.path, buffer: entry.body }); + } + return tree.read(onEntry); + }; + + tree.read(onEntry); + }); + }); +}; diff --git a/BuildServer/lib/task-processor.js b/BuildServer/lib/task-processor.js new file mode 100644 index 0000000..9191279 --- /dev/null +++ b/BuildServer/lib/task-processor.js @@ -0,0 +1,79 @@ +"use strict"; + +//TaskProcessor does not look like EventEmitter, so no need to extend EventEmitter and use `emit' here. +var TaskProcessor = function (task, outerProcessor, callback) { + if (!this) { + return new TaskProcessor(task); + } + + var self = this, + taskImpl = require('./tasks/' + task.type.match(/[\w\-]/g).join("")), + taskWorker = taskImpl(task.params, self), + errors = [], + process = function () { + taskWorker.process(); + }, + getOuterPrefix = function (prefix) { + return task.name + (prefix ? "/" + prefix : ""); + }, + onError = function (message, prefix) { + errors.push(message); + outerProcessor.onError(message, getOuterPrefix(prefix)); + }, + onWarn = function (message, prefix) { + outerProcessor.onWarn(message, getOuterPrefix(prefix)); + }, + onInfo = function (message, prefix) { + outerProcessor.onInfo(message, getOuterPrefix(prefix)); + }, + processTask = function (innerTask, innerCallback) { + var innerProcessor = new TaskProcessor(innerTask, self, innerCallback); + innerProcessor.process(); + }, + done = function () { + callback(errors.join("\r\n")); + }; + + self.process = process; + self.onError = onError; + self.onWarn = onWarn; + self.onInfo = onInfo; + self.processTask = processTask; + self.done = done; +}; + +exports.processTask = function (task, callback) { + var errors = {}, + warns = {}, + infos = {}, + messageProcessor = function (list) { + return function (message, prefix) { + var i, + parts = prefix.split("/"), + innerList = list; + + for (i = 0; i < parts.length; i += 1) { + innerList = (innerList[parts[i]] = innerList[parts[i]] || {}); + } + + innerList.$messages = innerList.$messages || []; + innerList.$messages.push(message); + + list.$allMessages = list.$allMessages || []; + list.$allMessages.push({ prefix: prefix, message: message }); + }; + }, + processor = new TaskProcessor(task, { + onError: messageProcessor(errors), + onWarn: messageProcessor(warns), + onInfo: messageProcessor(infos) + }, function (err) { + callback(err, { + errors: errors, + warns: warns, + infos: infos + }); + }); + + processor.process(); +}; diff --git a/BuildServer/lib/tasks/echo.js b/BuildServer/lib/tasks/echo.js new file mode 100644 index 0000000..4e23c55 --- /dev/null +++ b/BuildServer/lib/tasks/echo.js @@ -0,0 +1,21 @@ +"use strict"; + +module.exports = function (params, processor) { + return { + process: function () { + if (params.error) { + processor.onError(params.error); + } + + if (params.warn) { + processor.onWarn(params.warn); + } + + if (params.info) { + processor.onInfo(params.info); + } + + processor.done(); + } + }; +}; diff --git a/BuildServer/package.json b/BuildServer/package.json new file mode 100644 index 0000000..1a077a9 --- /dev/null +++ b/BuildServer/package.json @@ -0,0 +1,15 @@ +{ + "name": "micro-build-server", + "version": "0.0.1", + "private": true, + "scripts": { + "start": "forever -c node app.js" + }, + "dependencies": { + "express": "3.4.4", + "jade": "*", + "async": "~0.2.9", + "fs-extra": "~0.8.1", + "git-node": "~0.1.1" + } +} diff --git a/BuildServer/public/stylesheets/style.css b/BuildServer/public/stylesheets/style.css new file mode 100644 index 0000000..30e047d --- /dev/null +++ b/BuildServer/public/stylesheets/style.css @@ -0,0 +1,8 @@ +body { + padding: 50px; + font: 14px "Lucida Grande", Helvetica, Arial, sans-serif; +} + +a { + color: #00B7FF; +} \ No newline at end of file diff --git a/BuildServer/routes/index.js b/BuildServer/routes/index.js new file mode 100644 index 0000000..31369f4 --- /dev/null +++ b/BuildServer/routes/index.js @@ -0,0 +1,12 @@ + +/* + * GET home page. + */ + +exports.index = function(req, res){ + res.render('index', { title: 'Express' + req + "qq" }); +}; + +exports.postreceive = require('./postreceive'); +exports.manual = require('./manual'); +exports.status = require('./status'); diff --git a/BuildServer/routes/manual.js b/BuildServer/routes/manual.js new file mode 100644 index 0000000..6728de1 --- /dev/null +++ b/BuildServer/routes/manual.js @@ -0,0 +1,19 @@ +var builder = require('../lib/builder'); + +exports.get = function (req, res) { + res.render('manual'); +}; + +exports.post = function (req, res) { + var options = req.body; + options.app = req.app; + + builder.build(options, function (err, result) { + console.log("Done processing manual request"); + console.log("Error: " + err); + console.log("Result:"); + console.log(result); + res.render('manual-done', {err: err, result: result}); + //res.render("manual-done", { err: err, result: result }); + }); +}; diff --git a/BuildServer/routes/postreceive.js b/BuildServer/routes/postreceive.js new file mode 100644 index 0000000..9657597 --- /dev/null +++ b/BuildServer/routes/postreceive.js @@ -0,0 +1,29 @@ +var builder = require('../lib/builder'); + +/* + * POST from github + */ + +module.exports = function(req, res){ + if (!req.body || !req.body.payload) { + res.end(); + } + + var payload = JSON.parse(req.body.payload), + repository = payload.repository; + + builder.build({ + app: req.app, + url: repository.url, + owner: repository.owner.name, + reponame: repository.name, + rev: payload.after, + branch: payload.ref + }, function (err, result) { + console.log("Done processing request from GitHub"); + console.log("Error: " + err); + console.log("Result:"); + console.log(result); + res.end(); + }); +}; \ No newline at end of file diff --git a/BuildServer/routes/status.js b/BuildServer/routes/status.js new file mode 100644 index 0000000..11fd9dd --- /dev/null +++ b/BuildServer/routes/status.js @@ -0,0 +1,5 @@ +exports.image = function(req, res){ + console.log(req.headers); + res.setHeader('Content-Type', 'image/svg+xml'); + res.render('status-image', { title: 'Express' + req + "qq", status: "Error" }); +}; diff --git a/BuildServer/views/index.jade b/BuildServer/views/index.jade new file mode 100644 index 0000000..ef7b09f --- /dev/null +++ b/BuildServer/views/index.jade @@ -0,0 +1,5 @@ +extends layout + +block content + h1= title + p Welcome to #{title} \ No newline at end of file diff --git a/BuildServer/views/layout.jade b/BuildServer/views/layout.jade new file mode 100644 index 0000000..1b7b305 --- /dev/null +++ b/BuildServer/views/layout.jade @@ -0,0 +1,7 @@ +doctype 5 +html + head + title= title + link(rel='stylesheet', href='/stylesheets/style.css') + body + block content \ No newline at end of file diff --git a/BuildServer/views/manual-done.jade b/BuildServer/views/manual-done.jade new file mode 100644 index 0000000..a12cf17 --- /dev/null +++ b/BuildServer/views/manual-done.jade @@ -0,0 +1,9 @@ +extends layout + +block content + h1 Manual build request processed! + h2 Error + pre #{err} + h2 Result + pre + = JSON.stringify(result, null, 4) diff --git a/BuildServer/views/manual-done.jade.bak b/BuildServer/views/manual-done.jade.bak new file mode 100644 index 0000000..39b8ca0 --- /dev/null +++ b/BuildServer/views/manual-done.jade.bak @@ -0,0 +1,8 @@ +extends layout + +block content + h1 Manual build request processed! + h2 Error + pre #{err} + h2 Result + pre #{result} diff --git a/BuildServer/views/manual.jade b/BuildServer/views/manual.jade new file mode 100644 index 0000000..2641bf2 --- /dev/null +++ b/BuildServer/views/manual.jade @@ -0,0 +1,24 @@ +extends layout + +block content + h1 Manual build request + + form(method="POST") + table + tr + td URL + td: input(type="text", name="url", value="https://github.pos/igor-prokhorov/test-github-integration") + tr + td Owner + td: input(type="text", name="owner", value="igor-prokhorov") + tr + td Repository + td: input(type="text", name="reponame", value="test-github-integration") + tr + td Version + td: input(type="text", name="rev", value="0d0ace9a07a43d9c8db2eb4bc8182ac7bd5a25c9") + tr + td Branch + td: input(type="text", name="branch", value="refs/heads/master") + tr + td(colspan=2): input(type="submit", value="submit") diff --git a/BuildServer/views/status-image.jade b/BuildServer/views/status-image.jade new file mode 100644 index 0000000..a1f250a --- /dev/null +++ b/BuildServer/views/status-image.jade @@ -0,0 +1,8 @@ +doctype xml + +-var fills = {Error: "red", Warn: "yellow", OK: "green" } +-var colors = {Error: "white", Warn: "black", OK: "black" } + +svg(xmlns="http://www.w3.org/2000/svg", version="1.1", width="800px", height="100px") + rect(width="100%", height="100%", fill=fills[status]) + text(x=20, y=50, font-size=36, font-weight="bold", fill=colors[status]) Build status: #{status}