From faa0ee14bd187f6f5493fe084426cfdad4367e89 Mon Sep 17 00:00:00 2001 From: Karel Minarik Date: Sun, 16 Jun 2013 23:25:08 +0200 Subject: [PATCH] [UTIL] Added a source code generator using the JSON API specs The generator is based on [Thor](https://github.com/wycats/thor), a library/framework for command line applications. The generator will read the JSON API spec file(s), and generate the Ruby source code (one file per API endpoint) with correct module namespace, method names, and RDoc documentation. It will generate a test file for each API endpoint as well. Currently it only generates Ruby source, but can easily be extended and adapted to generate source code for other programming languages. Usage example: $ thor api:code:generate ../../api-spec/*.json --force --verbose --- rest-api-spec/utils/Gemfile | 2 + rest-api-spec/utils/Thorfile | 1 + rest-api-spec/utils/thor/generate_api.rb | 6 +- rest-api-spec/utils/thor/generate_source.rb | 117 ++++++++++++++++++ .../utils/thor/templates/ruby/method.erb | 59 +++++++++ .../utils/thor/templates/ruby/test.erb | 26 ++++ .../utils/thor/templates/ruby/test_helper.rb | 54 ++++++++ 7 files changed, 262 insertions(+), 3 deletions(-) create mode 100644 rest-api-spec/utils/thor/generate_source.rb create mode 100644 rest-api-spec/utils/thor/templates/ruby/method.erb create mode 100644 rest-api-spec/utils/thor/templates/ruby/test.erb create mode 100644 rest-api-spec/utils/thor/templates/ruby/test_helper.rb diff --git a/rest-api-spec/utils/Gemfile b/rest-api-spec/utils/Gemfile index aad88e80992..313d822986f 100644 --- a/rest-api-spec/utils/Gemfile +++ b/rest-api-spec/utils/Gemfile @@ -2,6 +2,8 @@ source 'https://rubygems.org' gem 'activesupport' gem 'rest-client' +gem 'coderay' +gem 'multi_json' gem 'thor' gem 'json' gem 'pry' diff --git a/rest-api-spec/utils/Thorfile b/rest-api-spec/utils/Thorfile index 887db65b6b2..2b8f53b4f88 100644 --- a/rest-api-spec/utils/Thorfile +++ b/rest-api-spec/utils/Thorfile @@ -1 +1,2 @@ require File.expand_path('./thor/generate_api') +require File.expand_path('./thor/generate_source') diff --git a/rest-api-spec/utils/thor/generate_api.rb b/rest-api-spec/utils/thor/generate_api.rb index 4122892775a..121b240e691 100644 --- a/rest-api-spec/utils/thor/generate_api.rb +++ b/rest-api-spec/utils/thor/generate_api.rb @@ -102,7 +102,7 @@ module Elasticsearch # * Assemble the JSON format for the API spec # class JsonGenerator < Thor - namespace 'api:generate' + namespace 'api:spec' include Thor::Actions @@ -110,14 +110,14 @@ module Elasticsearch # Usage: thor help api:generate:spec # - desc "spec", "Generate JSON API spec files from Elasticsearch source code" + desc "generate", "Generate JSON API spec files from Elasticsearch source code" method_option :force, type: :boolean, default: false, desc: 'Overwrite the output' method_option :verbose, type: :boolean, default: false, desc: 'Output more information' method_option :output, default: __root.join('tmp/out'), desc: 'Path to output directory' method_option :elasticsearch, default: __root.join('tmp/elasticsearch'), desc: 'Path to directory with Elasticsearch source code' method_option :crawl, type: :boolean, default: false, desc: 'Extract URLs from Elasticsearch website' - def spec + def generate self.class.source_root File.expand_path('../', __FILE__) @output = options[:output] diff --git a/rest-api-spec/utils/thor/generate_source.rb b/rest-api-spec/utils/thor/generate_source.rb new file mode 100644 index 00000000000..2002da15f55 --- /dev/null +++ b/rest-api-spec/utils/thor/generate_source.rb @@ -0,0 +1,117 @@ +# encoding: UTF-8 + +require 'thor' + +require 'pathname' +require 'active_support/core_ext/hash/deep_merge' +require 'active_support/inflector' +require 'multi_json' +require 'coderay' +require 'pry' + +module Elasticsearch + + module API + + # A command line application based on [Thor](https://github.com/wycats/thor), + # which will read the JSON API spec file(s), and generate + # the Ruby source code (one file per API endpoint) with correct + # module namespace, method names, and RDoc documentation, + # as well as test files for each endpoint. + # + # Currently it only generates Ruby source, but can easily be + # extended and adapted to generate source code for other + # programming languages. + # + class SourceGenerator < Thor + namespace 'api:code' + + include Thor::Actions + + __root = Pathname( File.expand_path('../../..', __FILE__) ) + + desc "generate ", "Generate source code and tests from the REST API JSON specification" + method_option :language, default: 'ruby', desc: 'Programming language' + method_option :force, type: :boolean, default: false, desc: 'Overwrite the output' + method_option :verbose, type: :boolean, default: false, desc: 'Output more information' + method_option :input, default: File.expand_path('../../../api-spec', __FILE__), desc: 'Path to directory with JSON API specs' + method_option :output, default: File.expand_path('../../../tmp/out', __FILE__), desc: 'Path to output directory' + + def generate(*files) + self.class.source_root File.expand_path('../', __FILE__) + + @input = Pathname(options[:input]) + @output = Pathname(options[:output]) + + # -- Test helper + copy_file "templates/ruby/test_helper.rb", @output.join('test').join('test_helper.rb') if options[:language] == 'ruby' + + files.each do |file| + @path = Pathname(file) + @json = MultiJson.load( File.read(@path) ) + @spec = @json.values.first + say_status 'json', @path, :yellow + # say_status 'JSON', @spec.inspect, options[:verbose] + + @full_namespace = @json.keys.first.split('.') + @namespace_depth = @full_namespace.size > 0 ? @full_namespace.size-1 : 0 + @module_namespace = @full_namespace[0, @namespace_depth] + @method_name = @full_namespace.last + + # -- Ruby files + + @path_to_file = @output.join('api').join( @module_namespace.join('/') ).join("#{@method_name}.rb") + + empty_directory @output.join('api').join( @module_namespace.join('/') ) + + template "templates/#{options[:language]}/method.erb", @path_to_file + + if options[:verbose] + colorized_output = CodeRay.scan_file(@path_to_file, :ruby).terminal + lines = colorized_output.split("\n") + say_status options[:language].downcase, + lines.first + "\n" + lines[1, lines.size].map { |l| ' '*14 + l }.join("\n"), + :yellow + end + + # --- Test files + + @test_directory = @output.join('test/api').join( @module_namespace.join('/') ) + @test_file = @test_directory.join("#{@method_name}_test.rb") + + empty_directory @test_directory + template "templates/#{options[:language]}/test.erb", @test_file + + if options[:verbose] + colorized_output = colorized_output = CodeRay.scan_file(@test_file, :ruby).terminal + lines = colorized_output.split("\n") + say_status options[:language].downcase, + lines.first + "\n" + lines[1, lines.size].map { |l| ' '*14 + l }.join("\n"), + :yellow + say '▬'*terminal_width + end + end + + # -- Tree output + + if options[:verbose] && `which tree > /dev/null 2>&1` + lines = `tree #{@output}`.split("\n") + say_status 'tree', + lines.first + "\n" + lines[1, lines.size].map { |l| ' '*14 + l }.join("\n") + end + end + + private + + # Create the hierarchy of directories based on API namespaces + # + def __create_directories(key, value) + unless value['documentation'] + empty_directory @output.join(key) + create_directory_hierarchy *value.to_a.first + end + end + + end + end +end diff --git a/rest-api-spec/utils/thor/templates/ruby/method.erb b/rest-api-spec/utils/thor/templates/ruby/method.erb new file mode 100644 index 00000000000..90583e50d3a --- /dev/null +++ b/rest-api-spec/utils/thor/templates/ruby/method.erb @@ -0,0 +1,59 @@ +module Elasticsearch + module API + <%- @module_namespace.each_with_index do |name, i| -%> + <%= ' '*i %>module <%= name.capitalize %> + <%- end -%> + <%= ' '*@namespace_depth %>module Actions + + <%= ' '*@namespace_depth %># <%= @spec['description'] || 'TODO: Description' %> + <%= ' '*@namespace_depth %># +<%# URL parts -%> + <%- @spec['url']['parts'].each do |name,info| -%> + <%- info['type'] = 'String' if info['type'] == 'enum' # Rename 'enums' to 'strings' -%> + <%= ' '*@namespace_depth + "# @option arguments [#{info['type'] ? info['type'].capitalize : 'String'}] :#{name} #{info['description']}" + ( info['required'] ? ' (*Required*)' : '' ) -%><%= " (options: #{info['options'].join(', ')})" if info['options'] -%> + <%= "\n" -%> + <%- end -%> +<%# Body -%> +<%= ' '*(@namespace_depth+3) + '# @option arguments [Hash] :body ' + (@spec['body']['description'] || 'TODO: Description') + "\n" if @spec['body'] -%> +<%# URL parameters -%> + <%- @spec['url']['params'].each do |name,info| -%> + <%- info['type'] = 'String' if info['type'] == 'enum' # Rename 'enums' to 'strings' -%> + <%= ' '*@namespace_depth + "# @option arguments [#{info['type'] ? info['type'].capitalize : 'String'}] :#{name} #{info['description']}" -%><%= " (options: #{info['options'].join(', ')})" if info['options'] -%> + <%= "\n" -%> + <%- end -%> + <%= ' '*@namespace_depth -%># +<%# Documentation link -%> + <%= ' '*@namespace_depth %># @see <%= @spec['documentation'] %> + <%= ' '*@namespace_depth %># +<%# Method definition -%> + <%= ' '*@namespace_depth -%>def <%= @method_name %>(arguments={}) +<%# Required arguments -%> + <%- @spec['url']['parts'].select { |name, info| info['required'] }.each do |name, info| -%> + <%= ' '*(@namespace_depth+1) + "raise ArgumentError, \"Required argument '#{name}' missing\" unless arguments[:#{name}]" + "\n" -%> + <%- end -%> +<%# Method, path, params, body -%> + <%= ' '*@namespace_depth %> method = '<%= @spec['methods'].first %>' + <%- unless @spec['url']['parts'].empty? -%> + <%= ' '*@namespace_depth %> path = "<%= @spec['url']['path'].split('/').compact.reject {|p| p =~ /^\s*$/}.map do |p| + p =~ /\{/ ? "\#\{arguments[:#{p.tr('{}', '')}]\}" : p + end.join('/') %>" + <%- else -%> + <%= ' '*@namespace_depth %> path = "<%= @spec['url']['path'] %>" + <%- end -%> + <%- unless @spec['url']['params'].keys.empty? -%> + <%= ' '*@namespace_depth %> params = arguments.select do |k,v| + <%= ' '*@namespace_depth %> [ :<%= @spec['url']['params'].keys.join(",\n#{' '*(@namespace_depth+6)}:") %> ].include?(k) + <%= ' '*@namespace_depth %> end + <%= ' '*@namespace_depth %> # Normalize Ruby 1.8 and Ruby 1.9 Hash#select behaviour + <%= ' '*@namespace_depth %> params = Hash[params] unless params.is_a?(Hash) + <%- end -%> + <%= ' '*@namespace_depth %> body = <%= @spec['body'].nil? ? 'nil' : '"TODO: Body specification missing"' %> +<%# Perform request %> + <%= ' '*@namespace_depth %> perform_request(method, path, params, body).body + <%= ' '*@namespace_depth %>end + <%- @namespace_depth.downto(1) do |i| -%> + <%= ' '*(i-1) %>end + <%- end if @namespace_depth > 0 -%> + end + end +end diff --git a/rest-api-spec/utils/thor/templates/ruby/test.erb b/rest-api-spec/utils/thor/templates/ruby/test.erb new file mode 100644 index 00000000000..0825d813c60 --- /dev/null +++ b/rest-api-spec/utils/thor/templates/ruby/test.erb @@ -0,0 +1,26 @@ +require 'test_helper' + +module Elasticsearch + module Test + class <%= @module_namespace.empty? ? @method_name.camelize : @module_namespace.map {|n| n.capitalize}.join + @method_name.camelize %>Test < ::Test::Unit::TestCase + + context "<%= @module_namespace.empty? ? '' : @module_namespace.map {|n| n.capitalize}.join + ': ' %><%= @method_name.humanize %>" do + subject { FakeClient.new(nil) } + + should "perform correct request" do + subject.expects(:perform_request).with do |method, url, params, body| + assert_equal 'FAKE', method + assert_equal 'test', url + assert_equal Hash.new, params + <%= @spec['body'].nil? ? 'assert_nil body' : 'assert_equal Hash.new, body' %> + true + end.returns(FakeResponse.new) + + subject.<%= @full_namespace.join('.') %> + end + + end + + end + end +end diff --git a/rest-api-spec/utils/thor/templates/ruby/test_helper.rb b/rest-api-spec/utils/thor/templates/ruby/test_helper.rb new file mode 100644 index 00000000000..6b822a1d2a0 --- /dev/null +++ b/rest-api-spec/utils/thor/templates/ruby/test_helper.rb @@ -0,0 +1,54 @@ +RUBY_1_8 = defined?(RUBY_VERSION) && RUBY_VERSION < '1.9' + +require 'rubygems' if RUBY_1_8 + +require 'simplecov' and SimpleCov.start { add_filter "/test|test_/" } if ENV["COVERAGE"] + +require 'test/unit' +require 'shoulda-context' +require 'mocha/setup' +require 'turn' unless ENV["TM_FILEPATH"] || ENV["NOTURN"] || RUBY_1_8 + +require 'require-prof' if ENV["REQUIRE_PROF"] +Dir[ File.expand_path('../../lib/elasticsearch/api/**/*.rb', __FILE__) ].each do |f| + puts 'Loading: ' + f.to_s if ENV['DEBUG'] + require f +end +RequireProf.print_timing_infos if ENV["REQUIRE_PROF"] + +module Elasticsearch + module Test + def __full_namespace(o) + o.constants.inject([o]) do |sum, c| + m = o.const_get(c.to_s.to_sym) + sum << __full_namespace(m).flatten if m.is_a?(Module) + sum + end.flatten + end; module_function :__full_namespace + + class FakeClient + # Include all "Actions" modules into the fake client + Elasticsearch::Test.__full_namespace(Elasticsearch::API).select { |m| m.to_s =~ /Actions$/ }.each do |m| + puts "Including: #{m}" if ENV['DEBUG'] + include m + end + + def perform_request(method, path, params, body) + puts "PERFORMING REQUEST:", "--> #{method.to_s.upcase} #{path} #{params} #{body}" + FakeResponse.new(200, 'FAKE', {}) + end + end + + FakeResponse = Struct.new(:status, :body, :headers) do + def status + values[0] || 200 + end + def body + values[1] || '{}' + end + def headers + values[2] || {} + end + end + end +end