sourcegraph · xizhao · Jan 14, 2014 · Jan 14, 2014 · Jan 14, 2014 · Jan 15, 2014
diff --git a/.bin/srclib-javascript b/.bin/srclib-javascript
@@ -0,0 +1,276 @@
+#!/usr/bin/env node
+
+var parser = require("nomnom");
+var glob = require("glob");
+var findpkgs = require("commonjs-findpkgs");
+var jsgAdapter = require("../lib/jsg_adapter")
+var npm = require("npm");
+var path = require("path");
+var execFile = require("child_process").execFile;
+var fs = require("fs");
+
+parser.command("scan-commonjs")
+  .option("repo", {
+    help: "repository URI"
+  })
+  .option("subdir", {
+    help: "subdirectory in repository"
+  })
+  .callback(function(opts) {
+    // TODO(sqs): support ScanIgnore from repo Srcfile (pass to findpkgs ignore param)
+    findpkgs(".", ["node_modules"], function(err, pkgs) {
+      if (err) {
+        console.error("Scanning failed:", err);
+        process.exit(1);
+      }
+
+      // omit package.json files that have syntax errors
+      pkgs = pkgs.filter(function(pkg) { return !pkg.error; });
+
+      // filter out undesirable packages (packages whose paths contain
+      // "node_modules", "test", etc.)
+      pkgs = pkgs.filter(function(pkg) {
+        // TODO(sqs): we probably want to process things in node_modules for
+        // local code, so remove this filter.
+        var pathComps = pkg.dir.split("/");
+        return pathComps.indexOf("node_modules") == -1 && pathComps.indexOf("test") == -1 && pathComps.indexOf("templates") == -1 && pathComps.indexOf("bower_components") == -1;
+      });
+
+      // filter out undesirable source files (minified files) from
+      // packages
+      pkgs.forEach(function(pkg) {
+        pkg.libFiles = pkg.libFiles.filter(function(f) {
+          return !/\.min\.js$/.test(f);
+        });
+      });
+
+      // automatically include files in the src dir (not just
+      // lib). this is somewhat common for client-side libs even
+      // though it is technically in violation of the commonjs spec.
+      // TODO(sqs): add this functionality (as an option?) to commonjs-findpkgs?
+      pkgs.forEach(function(pkg) {
+        var srcFiles = glob.sync(path.join(pkg.dir, 'src/**/*.js'));
+        if (srcFiles) pkg.libFiles.push.apply(pkg.libFiles, srcFiles);
+      });        
+
+      // automatically include files in the dir containing the main file (if it's a subdir)
+      // TODO(sqs): add this functionality (as an option?) to commonjs-findpkgs?
+      pkgs.forEach(function(pkg) {
+        if (pkg.package.main && pkg.package.main.indexOf('/') != -1) {
+          var files = glob.sync(path.join(pkg.dir, path.dirname(pkg.package.main), '**/*.js'));
+          if (files) pkg.libFiles.push.apply(pkg.libFiles, files);
+        }
+      });        
+
+      // convert from commonjs-findpkgs format to source unit
+      var srcunits = pkgs.map(function(pkg) {
+        // collect all deps
+        var allDeps = [{name: "node", "version": ""}];
+        function collectDeps(deps) {
+          if (deps) Object.keys(deps).forEach(function(name) { allDeps.push({name: name, version: deps[name]}); });
+        }
+        collectDeps(pkg.package.dependencies);
+
+        files = arrayUnique(pkg.libFiles.concat(pkg.testFiles).filter(function(f) { return fs.statSync(f).isFile(); }));
+
+        return {
+          Name: pkg.package.name,
+          Type: "CommonJSPackage",
+          Dir: pkg.dir,
+          Files: files,
+          Dependencies: allDeps,
+          Data: pkg.package,
+          Ops: {depresolve: null, graph: null},
+          Config: {
+            npmInstall: !!process.env.IN_DOCKER_CONTAINER,
+            jsg: {
+              plugins: {
+                node: {
+                  coreModulesDir: process.env.NODEJS_CORE_MODULES_DIR || "$(JSG_DIR)/testdata/node_core_modules",
+                }
+              },
+            },
+          },
+        };
+      });
+
+      console.log(JSON.stringify(srcunits, null, 2));
+    });
+  })
+  .help("scan for CommonJS packages");
+
+parser.command("depresolve")
+  .callback(function(opts) {
+    var stdin = process.stdin,
+    stdout = process.stdout,
+    inputChunks = [];
+
+    stdin.resume();
+    stdin.setEncoding("utf8");
+
+    stdin.on("data", function (chunk) {
+      inputChunks.push(chunk);
+    });
+
+    stdin.on("end", function () {
+      var inputJSON = inputChunks.join("");
+      var commonJSPackage = JSON.parse(inputJSON);
+
+      var remaining = commonJSPackage.Dependencies ? commonJSPackage.Dependencies.length : 0;
+      var resolutions = [];
+
+      if (remaining == 0) {
+        console.log('[]');
+        return;
+      }
+
+      commonJSPackage.Dependencies.forEach(function(dep) {
+        var i = commonJSPackage.Dependencies.indexOf(dep);
+        resolutions[i] = {Raw: dep};
+        var spec = dep.name+(dep.version ? "@"+dep.version : "");
+        npm.load(function(err, npm) {
+          if (err) {
+            console.error("npm.load failed: ", err);
+            process.exit(1)
+          }
+
+          npm.commands.view(["--silent", spec], function(err,data) {
+            if (hardcodedRepoURLs[spec]) {
+              data = {"": {}}
+              err = null
+            }
+
+            if (err) {
+              resolutions[i].Error = "error occurred while resolving '" + spec + "': " + JSON.stringify(err);
+            } else if (!data || Object.keys(data).length == 0) {
+              resolutions[i].Error = "no npm package found with spec " + JSON.stringify(spec);
+            } else {
+              // Choose a version at random if it spits back a lot (TODO(sqs):
+              // this is stupid, we want the version that `npm install` would
+              // install, not just any random version, especially when
+              // version=='*').
+              var chooseVersion = Object.keys(data)[0];
+              var info = data[chooseVersion];
+
+              var repoURL = "";
+              if (hardcodedRepoURLs[spec]) {
+                repoURL = hardcodedRepoURLs[spec];
+              } else if (info.repository) {
+                if (typeof info.repository == "string") repoURL = info.repository;
+                else repoURL = info.repository.url;
+              }
+
+              resolutions[i].Target = {
+                ToRepoCloneURL: repoURL,
+	        ToUnit: info.name,
+	        ToUnitType: "CommonJSPackage",
+	        ToVersionString: info.version,
+              };
+            }
+
+            remaining--;
+            if (remaining == 0) {
+              // done resolving all
+              console.log(JSON.stringify(resolutions, null, 2));
+            }
+          });
+        });
+      });
+    });
+  })
+  .help("resolves dependencies of a CommonJS package");
+
+var hardcodedRepoURLs = {
+    "node": "https://github.com/joyent/node.git",
+};
+
+parser.command("graph")
+  .callback(function(opts) {
+    var stdin = process.stdin,
+    stdout = process.stdout,
+    inputChunks = [];
+
+    stdin.resume();
+    stdin.setEncoding("utf8");
+
+    stdin.on("data", function (chunk) {
+      inputChunks.push(chunk);
+    });
+
+    stdin.on("end", function () {
+      var inputJSON = inputChunks.join("");
+      var commonJSPackage = JSON.parse(inputJSON);
+
+      function graph() {
+        if (commonJSPackage.Files.length == 0) {
+          console.log("{}");
+          process.exit(0);
+        }
+        jsgAdapter.run(".", commonJSPackage, function(err, graphData) {
+          if (err) {
+            console.error("Graphing " + commonJSPackage.Name + " failed:", err);
+            process.exit(1);
+          }
+          console.log(JSON.stringify(graphData, null, 2));
+        });
+      }
+
+      // TODO(sqs) install npm deps if we're in a docker container, since we can't rely on
+      // them to have been installed locally. we need to account for the fact
+      // that we've mounted the source volume readonly, so we can't just run
+      // `npm install`.
+      if (process.env.IN_DOCKER_CONTAINER && commonJSPackage.Config.npmInstall) {
+        // running `npm install` in the /src dir fails because it's mounted
+        // readonly. but `npm install -g` in the /src dir does nothing. so we
+        // need to `npm install /src` from a dir other than /src to get `npm
+        // install` to do anything.
+
+        var packageDir = path.join("/src", commonJSPackage.Dir);
+        console.error("Installing npm package in dir", packageDir, "...");
+        execFile("npm", ["install", "--ignore-scripts", packageDir], {cwd: "/"}, function(err, stdout, stderr) {
+          if (stderr) console.error(stderr);
+          if (stdout) console.error(stdout);
+          if (err) {
+            console.error("warning: npm install in Docker container failed: ", err, "(continuing with graphing...)");
+          }
+          graph();
+        });
+
+
+        // It's apparently impossible to silence npm install output when using
+        // npm.install. :(
+        // var oldcwd = process.cwd();
+        // process.chdir(path.join(oldcwd, ".."));
+        // console.error("npm installing packages in Docker container...");
+        // npm.load({loglevel: "silent"}, function(err, npm) {
+        //   if (err) {
+        //     console.error("npm.load in Docker container failed: ", err);
+        //     process.exit(1)
+        //   }
+        //   // TODO(sqs): only works for top-level package.jsons. we actually need
+        //   // "/src/${dirname(package.json path)", which also requires us to
+        //   // track the path to the package.json.
+        //   npm.commands.install(["/src"], function(err,data) {
+        //     if (err) {
+        //       console.error("npm install in Docker container failed: ", err);
+        //       process.exit(1)
+        //     }
+        //     process.chdir(oldcwd);
+        //     graph();
+        //   });
+        // });
+      } else {
+        graph();
+      }
+    });
+  })
+  .help("graph a CommonJS package using jsg/tern");
+
+parser.parse();
+
+var arrayUnique = function(a) {
+    return a.reduce(function(p, c) {
+        if (p.indexOf(c) < 0) p.push(c);
+        return p;
+    }, []);
+};
diff --git a/.dockerignore b/.dockerignore
@@ -0,0 +1,5 @@
+.git
+testdata
+__old
+.srclib-cache
+
diff --git a/.gitignore b/.gitignore
@@ -0,0 +1 @@
+testdata/actual/
diff --git a/.gitmodules b/.gitmodules
@@ -0,0 +1,12 @@
+[submodule "testdata/case/javascript-nodejs-sample-0"]
+	path = testdata/case/javascript-nodejs-sample-0
+	url = https://github.com/sgtest/javascript-nodejs-sample-0.git
+[submodule "testdata/case/javascript-nodejs-xrefs-0"]
+	path = testdata/case/javascript-nodejs-xrefs-0
+	url = https://github.com/sgtest/javascript-nodejs-xrefs-0.git
+[submodule "testdata/case/minimal_nodejs_stdlib"]
+	path = testdata/case/minimal_nodejs_stdlib
+	url = https://github.com/sgtest/minimal_nodejs_stdlib.git
+[submodule "testdata/case/js-misc"]
+	path = testdata/case/js-misc
+	url = https://github.com/sgtest/js-misc.git
diff --git a/.travis.yml b/.travis.yml
@@ -0,0 +1,12 @@
+language: node_js
+
+node_js:
+  - "0.11"
+  - "0.10"
+
+install:
+  - sudo wget -NO /tmp/src.zip 'https://api.equinox.io/1/Applications/ap_BQxVz1iWMxmjQnbVGd85V58qz6/Updates/Asset/src-0.0.22.zip?os=linux&arch=amd64&channel=stable'
+  - sudo unzip /tmp/src*.zip -d /usr/local/bin
+  - sudo mv /usr/local/bin/src-* /usr/local/bin/src
+  - sudo chmod +x /usr/local/bin/src
+  - src toolchain add sourcegraph.com/sourcegraph/srclib-javascript
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -0,0 +1,71 @@
+# How to Contribute
+
+Sourcegraph projects are [BSD licensed](LICENSE) and accept contributions via
+GitHub pull requests. This document outlines some of the conventions on
+development workflow, commit message formatting, contact points and other
+resources to make it easier to get your contribution accepted.
+
+# Certificate of Origin
+
+By contributing to this project you agree to the Developer Certificate of Origin
+(DCO). This document was created by the Linux Kernel community and is a simple
+statement that you, as a contributor, have the legal right to make the
+contribution. See the [DCO](DCO) file for details.
+
+# Email and Chat
+
+The project currently uses the general Sourcegraph email list and IRC channel:
+- Email: [srclib-dev](https://groups.google.com/forum/#!forum/srclib-dev)
+- IRC: #[srclib](irc://irc.freenode.org:6667/#srclib) IRC channel on freenode.org
+
+Please avoid emailing maintainers found in the MAINTAINERS file directly. They
+are very busy and read the mailing lists.
+
+## Getting Started
+
+- Fork the repository on GitHub
+- Read the [README](README.md) for build and test instructions
+- Play with the project, submit bugs, submit patches!
+
+## Contribution Flow
+
+This is a rough outline of what a contributor's workflow looks like:
+
+- Create a topic branch from where you want to base your work (usually master).
+- Make commits of logical units.
+- Make sure your commit messages are in the proper format (see below).
+- Push your changes to a topic branch in your fork of the repository.
+- Make sure the tests pass, and add any new tests as appropriate.
+- Submit a pull request to the original repository.
+
+Thanks for your contributions!
+
+### Format of the Commit Message
+
+We follow a rough convention for commit messages that is designed to answer two
+questions: what changed and why. The subject line should feature the what and
+the body of the commit should describe the why.
+
+```
+scripts: add the test-cluster command
+
+this uses tmux to setup a test cluster that you can easily kill and
+start for debugging.
+
+Fixes #38
+```
+
+The format can be described more formally as follows:
+
+```
+<subsystem>: <what changed>
+<BLANK LINE>
+<why this change was made>
+<BLANK LINE>
+<footer>
+```
+
+The first line is the subject and should be no longer than 70 characters, the
+second line is always blank, and other lines should be wrapped at 80 characters.
+This allows the message to be easier to read on Sourcegraph and GitHub as well
+as in various git tools.