Merge remote-tracking branch 'origin/master' into rewrite-readme

* origin/master: (104 commits)
  Added shebang sample for Pike.
  Added interpreter "pike" for Pike.
  Add support for FXML files.
  Add support for Turtle and SPARQL
  Fixed issues for web ontology to pass tests
  Added Web Ontology Language Support
  Simplify blob tests
  Use the original FileBlob path for filesystem access
  Sample sagews file, as requested
  Update languages.yml with *.sagews
  New grammar for Racket
  Remove grammar for Racket
  Modifying BlobHelper and FileBlob to use path
  Sample file for .cmake.in
  Restore the .cmake.in extension.
  More CMake samples.
  Updating file regex to support unlicense.txt
  Updating ref to include license
  Remove pry
  Start using path with LazyBlob
  ...

Conflicts:
	CONTRIBUTING.md
	README.md

Author: bkeepers

.gitmodules

@@ -82,9 +82,6 @@
 [submodule "vendor/grammars/language-python"]
 	path = vendor/grammars/language-python
 	url = https://github.com/atom/language-python
-[submodule "vendor/grammars/language-sass"]
-	path = vendor/grammars/language-sass
-	url = https://github.com/atom/language-sass
 [submodule "vendor/grammars/language-shellscript"]
 	path = vendor/grammars/language-shellscript
 	url = https://github.com/atom/language-shellscript
@@ -115,9 +112,6 @@
 [submodule "vendor/grammars/nesC.tmbundle"]
 	path = vendor/grammars/nesC.tmbundle
 	url = https://github.com/cdwilson/nesC.tmbundle
-[submodule "vendor/grammars/racket-tmbundle"]
-	path = vendor/grammars/racket-tmbundle
-	url = https://github.com/christophevg/racket-tmbundle
 [submodule "vendor/grammars/haxe-sublime-bundle"]
 	path = vendor/grammars/haxe-sublime-bundle
 	url = https://github.com/clemos/haxe-sublime-bundle
@@ -244,9 +238,6 @@
 [submodule "vendor/grammars/abap.tmbundle"]
 	path = vendor/grammars/abap.tmbundle
 	url = https://github.com/pvl/abap.tmbundle
-[submodule "vendor/grammars/Scalate.tmbundle"]
-	path = vendor/grammars/Scalate.tmbundle
-	url = https://github.com/scalate/Scalate.tmbundle
 [submodule "vendor/grammars/mercury-tmlanguage"]
 	path = vendor/grammars/mercury-tmlanguage
 	url = https://github.com/sebgod/mercury-tmlanguage
@@ -486,7 +477,7 @@
 	url = https://github.com/vkostyukov/kotlin-sublime-package
 [submodule "vendor/grammars/c.tmbundle"]
 	path = vendor/grammars/c.tmbundle
-	url = https://github.com/vmg/c.tmbundle
+	url = https://github.com/textmate/c.tmbundle
 [submodule "vendor/grammars/zephir-sublime"]
 	path = vendor/grammars/zephir-sublime
 	url = https://github.com/vmg/zephir-sublime
@@ -509,7 +500,6 @@
 [submodule "vendor/grammars/sublime-mask"]
 	path = vendor/grammars/sublime-mask
 	url = https://github.com/tenbits/sublime-mask
-	branch = release
 [submodule "vendor/grammars/sublime_cobol"]
 	path = vendor/grammars/sublime_cobol
 	url = https://bitbucket.org/bitlang/sublime_cobol
@@ -520,3 +510,42 @@
 [submodule "vendor/grammars/IDL-Syntax"]
 	path = vendor/grammars/IDL-Syntax
 	url = https://github.com/andik/IDL-Syntax
+[submodule "vendor/grammars/sas.tmbundle"]
+	path = vendor/grammars/sas.tmbundle
+	url = https://github.com/rpardee/sas.tmbundle
+[submodule "vendor/grammars/atom-salt"]
+	path = vendor/grammars/atom-salt
+	url = https://github.com/saltstack/atom-salt
+[submodule "vendor/grammars/Scalate.tmbundle"]
+	path = vendor/grammars/Scalate.tmbundle
+	url = https://github.com/scalate/Scalate.tmbundle
+[submodule "vendor/grammars/Elm.tmLanguage"]
+	path = vendor/grammars/Elm.tmLanguage
+	url = https://github.com/deadfoxygrandpa/Elm.tmLanguage
+[submodule "vendor/grammars/sublime-bsv"]
+	path = vendor/grammars/sublime-bsv
+	url = https://github.com/thotypous/sublime-bsv
+[submodule "vendor/grammars/AutoHotkey"]
+	path = vendor/grammars/AutoHotkey
+	url = https://github.com/robertcollier4/AutoHotkey
+[submodule "vendor/grammars/Sublime-HTTP"]
+	path = vendor/grammars/Sublime-HTTP
+	url = https://github.com/httpspec/sublime-highlighting
+[submodule "vendor/grammars/sass-textmate-bundle"]
+	path = vendor/grammars/sass-textmate-bundle
+	url = https://github.com/nathos/sass-textmate-bundle
+[submodule "vendor/grammars/carto-atom"]
+	path = vendor/grammars/carto-atom
+	url = https://github.com/yohanboniface/carto-atom
+[submodule "vendor/grammars/Sublime-Nit"]
+	path = vendor/grammars/Sublime-Nit
+	url = https://github.com/R4PaSs/Sublime-Nit
+[submodule "vendor/grammars/language-hy"]
+	path = vendor/grammars/language-hy
+	url = https://github.com/rwtolbert/language-hy
+[submodule "vendor/grammars/Racket"]
+	path = vendor/grammars/Racket
+	url = https://github.com/soegaard/racket-highlight-for-github
+[submodule "vendor/grammars/turtle.tmbundle"]
+	path = vendor/grammars/turtle.tmbundle
+	url = https://github.com/peta/turtle.tmbundle

.travis.yml

@@ -1,12 +1,5 @@
-before_install:
-  - git fetch origin master:master
-  - git fetch origin v2.0.0:v2.0.0
-  - git fetch origin test/attributes:test/attributes
-  - git fetch origin test/master:test/master
-  - sudo apt-get install libicu-dev -y
-  - git submodule init
-  - git submodule sync --quiet
-  - script/fast-submodule-update
+sudo: false
+before_install: script/travis/before_install
 rvm:
   - 1.9.3
   - 2.0.0
@@ -16,3 +9,4 @@ notifications:
   disabled: true
 git:
   submodules: false
+cache: bundler

CONTRIBUTING.md

@@ -13,10 +13,18 @@ To add support for a new language:
 0. Add an entry for your language to [`languages.yml`][languages].
 0. Add a grammar for your language. Please only add grammars that have a license that permits redistribution.
   0. Add your grammar as a submodule: `git submodule add https://github.com/JaneSmith/MyGrammar vendor/grammars/MyGrammar`.
-  0. Add your grammar to [`grammars.yml`][grammars] by running `script/download-grammars --add vendor/grammars/MyGrammar`.
+  0. Add your grammar to [`grammars.yml`][grammars] by running `script/convert-grammars --add vendor/grammars/MyGrammar`.
 0. Add samples for your language to the [samples directory][samples] in the correct subdirectory.
 0. Open a pull request, linking to a [GitHub search result](https://github.com/search?utf8=%E2%9C%93&q=extension%3Aboot+NOT+nothack&type=Code&ref=searchresults) showing in-the-wild usage.
 
+In addition, if your new language defines an extension that's already listed in [`languages.yml`][languages] (such as `.foo`) then sometimes a few more steps will need to be taken:
+
+0. Make sure that example `.foo` files are present in the [samples directory][samples] for each language that uses `.foo`.
+0. Test the performance of the Bayesian classifier with a relatively large number (1000s) of sample `.foo` files. (ping @arfon or @bkeepers to help with this) to ensure we're not misclassifying files.
+0. If the Bayesian classifier does a bad job with the sample `.foo` files then a [heuristic](https://github.com/github/linguist/blob/master/lib/linguist/heuristics.rb) may need to be written to help.
+
+Remember, the goal here is to try and avoid false positives!
+
 ## Fixing syntax highlighting
 
 Syntax highlighting in GitHub is performed using TextMate-compatible grammars. These are the same grammars that TextMate, Sublime Text and Atom use. Every language in `languages.yml` is mapped to its corresponding TM `scope`. This scope will be used when picking up a grammar for highlighting.
@@ -49,6 +57,7 @@ If you are the current maintainer of this gem:
 
 0. Create a branch for the release: `git checkout -b cut-release-vxx.xx.xx`
 0. Make sure your local dependencies are up to date: `script/bootstrap`
+0. If grammar submodules have not been updated recently, update them: `git submodule update --remote && git commit -a`
 0. Ensure that samples are updated: `bundle exec rake samples`
 0. Ensure that tests are green: `bundle exec rake test`
 0. Bump gem version in `lib/linguist/version.rb`, [like this](https://github.com/github/linguist/commit/8d2ea90a5ba3b2fe6e1508b7155aa4632eea2985).

Gemfile

@@ -1,5 +1,4 @@
 source 'https://rubygems.org'
 gemspec :name => "github-linguist"
 gemspec :name => "github-linguist-grammars"
-gem 'test-unit', require: false if RUBY_VERSION >= '2.2'
 gem 'byebug' if RUBY_VERSION >= '2.0'

Rakefile

@@ -48,7 +48,7 @@ end
 
 task :build_grammars_gem do
   rm_rf "grammars"
-  sh "script/download-grammars"
+  sh "script/convert-grammars"
   sh "gem", "build", "github-linguist-grammars.gemspec"
 end
 

github-linguist.gemspec

@@ -18,6 +18,7 @@ Gem::Specification.new do |s|
   s.add_dependency 'mime-types',      '>= 1.19'
   s.add_dependency 'rugged',          '~> 0.22.0b4'
 
+  s.add_development_dependency 'minitest', '>= 5.0'
   s.add_development_dependency 'mocha'
   s.add_development_dependency 'pry'
   s.add_development_dependency 'rake'

grammars.yml

@@ -24,13 +24,17 @@ vendor/grammars/Agda.tmbundle:
 - source.agda
 vendor/grammars/Alloy.tmbundle:
 - source.alloy
+vendor/grammars/AutoHotkey:
+- source.ahk
 vendor/grammars/ColdFusion:
 - source.cfscript
 - source.cfscript.cfc
 - text.cfml.basic
 - text.html.cfm
 vendor/grammars/Docker.tmbundle:
 - source.dockerfile
+vendor/grammars/Elm.tmLanguage:
+- source.elm
 vendor/grammars/Handlebars:
 - text.html.handlebars
 vendor/grammars/IDL-Syntax:
@@ -45,13 +49,15 @@ vendor/grammars/LiveScript.tmbundle:
 vendor/grammars/NSIS:
 - source.nsis
 vendor/grammars/NimLime:
-- source.nimrod
-- source.nimrod_filter
-- source.nimrodcfg
+- source.nim
+- source.nim_filter
+- source.nimcfg
 vendor/grammars/PHP-Twig.tmbundle:
 - text.html.twig
 vendor/grammars/RDoc.tmbundle:
 - text.rdoc
+vendor/grammars/Racket:
+- source.racket
 vendor/grammars/SCSS.tmbundle:
 - source.scss
 vendor/grammars/Scalate.tmbundle:
@@ -64,6 +70,8 @@ vendor/grammars/Stata.tmbundle:
 - source.stata
 vendor/grammars/Sublime-Coq:
 - source.coq
+vendor/grammars/Sublime-HTTP:
+- source.httpspec
 vendor/grammars/Sublime-Inform:
 - source.Inform7
 vendor/grammars/Sublime-Lasso:
@@ -72,6 +80,8 @@ vendor/grammars/Sublime-Logos:
 - source.logos
 vendor/grammars/Sublime-Loom:
 - source.loomscript
+vendor/grammars/Sublime-Nit:
+- source.nit
 vendor/grammars/Sublime-QML:
 - source.qml
 vendor/grammars/Sublime-REBOL:
@@ -115,6 +125,9 @@ vendor/grammars/asp.tmbundle:
 vendor/grammars/assembly.tmbundle:
 - objdump.x86asm
 - source.x86asm
+vendor/grammars/atom-salt:
+- source.python.salt
+- source.yaml.salt
 vendor/grammars/autoitv3-tmbundle:
 - source.autoit.3
 vendor/grammars/awk-sublime:
@@ -131,6 +144,8 @@ vendor/grammars/c.tmbundle:
 - source.c.platform
 vendor/grammars/capnproto.tmbundle:
 - source.capnp
+vendor/grammars/carto-atom:
+- source.css.mss
 vendor/grammars/ceylon-sublimetext:
 - module.ceylon
 - source.ceylon
@@ -248,16 +263,16 @@ vendor/grammars/language-csharp:
 - source.nant-build
 vendor/grammars/language-gfm:
 - source.gfm
+vendor/grammars/language-hy:
+- source.hy
 vendor/grammars/language-javascript:
 - source.js
 - source.js.regexp
 vendor/grammars/language-python:
 - source.python
 - source.regexp.python
+- text.python.console
 - text.python.traceback
-vendor/grammars/language-sass:
-- source.css.scss
-- source.sass
 vendor/grammars/language-shellscript:
 - source.shell
 - text.shell-session
@@ -349,8 +364,6 @@ vendor/grammars/python-django.tmbundle:
 vendor/grammars/r.tmbundle:
 - source.r
 - text.tex.latex.rd
-vendor/grammars/racket-tmbundle:
-- source.racket
 vendor/grammars/restructuredtext.tmbundle:
 - text.restructuredtext
 vendor/grammars/ruby-haml.tmbundle:
@@ -366,6 +379,11 @@ vendor/grammars/ruby-slim.tmbundle:
 vendor/grammars/ruby.tmbundle:
 - source.ruby
 - text.html.erb
+vendor/grammars/sas.tmbundle:
+- source.SASLog
+- source.sas
+vendor/grammars/sass-textmate-bundle:
+- source.sass
 vendor/grammars/scala.tmbundle:
 - source.sbt
 - source.scala
@@ -386,6 +404,8 @@ vendor/grammars/sublime-befunge:
 - source.befunge
 vendor/grammars/sublime-better-typescript:
 - source.ts
+vendor/grammars/sublime-bsv:
+- source.bsv
 vendor/grammars/sublime-cirru:
 - source.cirru
 vendor/grammars/sublime-glsl:
@@ -432,6 +452,9 @@ vendor/grammars/thrift.tmbundle:
 - source.thrift
 vendor/grammars/toml.tmbundle:
 - source.toml
+vendor/grammars/turtle.tmbundle:
+- source.sparql
+- source.turtle
 vendor/grammars/verilog.tmbundle:
 - source.verilog
 vendor/grammars/x86-assembly-textmate-bundle:

lib/linguist/blob_helper.rb

@@ -99,7 +99,7 @@ def disposition
       elsif name.nil?
         "attachment"
       else
-        "attachment; filename=#{EscapeUtils.escape_url(File.basename(name))}"
+        "attachment; filename=#{EscapeUtils.escape_url(name)}"
       end
     end
 
@@ -233,7 +233,7 @@ def viewable?
     #
     # Return true or false
     def vendored?
-      name =~ VendoredRegexp ? true : false
+      path =~ VendoredRegexp ? true : false
     end
 
     # Public: Get each line of data
@@ -301,7 +301,7 @@ def sloc
     #
     # Return true or false
     def generated?
-      @_generated ||= Generated.generated?(name, lambda { data })
+      @_generated ||= Generated.generated?(path, lambda { data })
     end
 
     # Public: Detects the Language of the blob.

lib/linguist/file_blob.rb

@@ -3,7 +3,7 @@
 module Linguist
   # A FileBlob is a wrapper around a File object to make it quack
   # like a Grit::Blob. It provides the basic interface: `name`,
-  # `data`, and `size`.
+  # `data`, `path` and `size`.
   class FileBlob
     include BlobHelper
 
@@ -14,43 +14,50 @@ class FileBlob
     #
     # Returns a FileBlob.
     def initialize(path, base_path = nil)
-      @path = path
-      @name = base_path ? path.sub("#{base_path}/", '') : path
+      @fullpath = path
+      @path = base_path ? path.sub("#{base_path}/", '') : path
     end
 
     # Public: Filename
     #
     # Examples
     #
-    #   FileBlob.new("/path/to/linguist/lib/linguist.rb").name
+    #   FileBlob.new("/path/to/linguist/lib/linguist.rb").path
     #   # =>  "/path/to/linguist/lib/linguist.rb"
     #
     #   FileBlob.new("/path/to/linguist/lib/linguist.rb",
-    #                "/path/to/linguist").name
+    #                "/path/to/linguist").path
     #   # =>  "lib/linguist.rb"
     #
     # Returns a String
-    attr_reader :name
+    attr_reader :path
 
     # Public: Read file permissions
     #
     # Returns a String like '100644'
     def mode
-      File.stat(@path).mode.to_s(8)
+      File.stat(@fullpath).mode.to_s(8)
+    end
+
+    # Public: File name
+    #
+    # Returns a String
+    def name
+      File.basename(@fullpath)
     end
 
     # Public: Read file contents.
     #
     # Returns a String.
     def data
-      File.read(@path)
+      File.read(@fullpath)
     end
 
     # Public: Get byte size
     #
     # Returns an Integer.
     def size
-      File.size(@path)
+      File.size(@fullpath)
     end
 
     # Public: Get file extension.
@@ -67,7 +74,7 @@ def extension
     #
     # Returns an Array
     def extensions
-      basename, *segments = File.basename(name).split(".")
+      basename, *segments = name.split(".")
 
       segments.map.with_index do |segment, index|
         "." + segments[index..-1].join(".")