Move parsing of Luna client header files to gem installation

In the pkcs11 base gem, we ship the PKCS#11 standard header files and link to the PKCS#11 ABI (not the API).

Due to copyright constraints we're not allowed to distribute Luna client header files in the gem as we do in the pkcs11 gem.
Instead we compile with the header files provided by the Luna client package, so we're using the API (not the ABI).

So far we distibuted the struct names and constant name/value pairs extracted from the header files at build time.
This however leads to inconsistencies between generated files at build time and the header files at install time, if the Luna client versions differ.
Moving the parsers to extconf.rb ensures that generated structs and constants fit to the header files that are included by the compiler.

Author: larskanis

pkcs11_luna/Manifest.txt

@@ -3,22 +3,22 @@
 Manifest.txt
 README_LUNA.rdoc
 Rakefile
-ext/extconf.rb
-ext/generate_constants.rb
-ext/generate_structs.rb
-ext/pk11l.c
-lib/pkcs11_luna.rb
-lib/pkcs11_luna/extensions.rb
-test/luna_helper.rb
-test/app_id_helper.rb
-test/test_pkcs11_luna.rb
-test/test_pkcs11_luna_crypt.rb
 examples/config.rb
 examples/derive_aes_ecdh_key.rb
-examples/sign_verify.rb
 examples/encrypt_decrypt_aes.rb
 examples/encrypt_decrypt_rsa.rb
 examples/mechanism_list.rb
 examples/multithread.rb
 examples/objects_list.rb
+examples/sign_verify.rb
 examples/slot_info.rb
+ext/extconf.rb
+ext/generate_luna_constants.rb
+ext/generate_luna_structs.rb
+ext/pk11l.c
+lib/pkcs11_luna.rb
+lib/pkcs11_luna/extensions.rb
+test/app_id_helper.rb
+test/luna_helper.rb
+test/test_pkcs11_luna.rb
+test/test_pkcs11_luna_crypt.rb

pkcs11_luna/Rakefile

@@ -7,26 +7,28 @@ require 'rake/extensiontask'
 require 'rbconfig'
 
 LUNA_INCLUDE_DIR = ENV['LUNA_INCLUDE_DIR'] || '/usr/safenet/lunaclient/samples/include'
-RUBY_PKCS11_EXT_DIR = File.expand_path('../ext')
+RUBY_PKCS11_DIR = File.expand_path('..')
 
 
-GENERATED_FILES = [
-  'ext/pk11l_struct_impl.inc',
-  'ext/pk11l_struct_def.inc',
-  'ext/pk11l_const_def.inc',
-  'ext/pk11l_struct.doc',
+SHARED_FILES = [
   'ext/pk11_struct_macros.h',
   'ext/pk11_const_macros.h',
   'ext/pk11_version.h',
+  'ext/generate_structs.rb',
+  'ext/generate_constants.rb',
+]
+GENERATED_FILES = [
+  "ext/std_structs.rb"
 ]
 
+CLEAN.include SHARED_FILES
 CLEAN.include GENERATED_FILES
 CLEAN.include 'lib/pkcs11_luna_ext.so'
 CLEAN.include 'tmp'
 CLEAN.include 'examples/output'
 
 def pkcs11_version
-  file = File.join(RUBY_PKCS11_EXT_DIR, 'pk11_version.h')
+  file = File.join(RUBY_PKCS11_DIR, 'ext/pk11_version.h')
   version_re = /VERSION += +([\"\'])([\d][\d\w\.]+)\1/
   File.read_utf(file)[version_re, 2]
 end
@@ -46,7 +48,7 @@ hoe = Hoe.spec 'pkcs11_luna' do
   self.extra_rdoc_files << self.readme_file << 'ext/pk11l.c'
   spec_extras[:extensions] = 'ext/extconf.rb'
   spec_extras[:files] = File.read_utf("Manifest.txt").split(/\r?\n\r?/)
-  spec_extras[:files] += GENERATED_FILES
+  spec_extras[:files] += SHARED_FILES + GENERATED_FILES
   spec_extras[:required_ruby_version] = '>= 2.2.0'
 end
 
@@ -58,29 +60,20 @@ Rake::ExtensionTask.new('pkcs11_luna_ext', hoe.spec) do |ext|
   ext.config_options << "--with-luna-dir-include=\"#{LUNA_INCLUDE_DIR}\""
 end
 
-def copy_from_base_task(filename)
-  file File.join('ext', filename) => File.join(RUBY_PKCS11_EXT_DIR, filename) do |t|
+# Add shared file from base pkcs11 gem
+SHARED_FILES.each do |filename|
+  file filename => File.join(RUBY_PKCS11_DIR, filename) do |t|
     cp t.prerequisites.first, t.name, verbose: true
   end
+  file 'ext/extconf.rb' => filename
 end
 
-copy_from_base_task 'pk11_struct_macros.h'
-copy_from_base_task 'pk11_const_macros.h'
-copy_from_base_task 'pk11_version.h'
-
-HEADER_FILES = "#{LUNA_INCLUDE_DIR}/RSA/pkcs11t.h #{LUNA_INCLUDE_DIR}/cryptoki_v2.h"
-
-file 'ext/extconf.rb' => ['ext/pk11l_struct_def.inc', 'ext/pk11l_const_def.inc', 'ext/pk11_struct_macros.h', 'ext/pk11_const_macros.h', 'ext/pk11_version.h']
-file 'ext/pk11l_struct_def.inc' => 'ext/generate_structs.rb' do
-  sh "#{RbConfig::CONFIG['ruby_install_name']} -I../lib ext/generate_structs.rb --def ext/pk11l_struct_def.inc --impl ext/pk11l_struct_impl.inc --doc ext/pk11l_struct.doc #{HEADER_FILES}"
-end
-file 'ext/pk11l_struct_impl.inc' => 'ext/pk11l_struct_def.inc'
-file 'ext/pk11l_struct.doc' => 'ext/pk11l_struct_def.inc'
+file "ext/std_structs.rb" do |t|
+  require "pkcs11"
+  std_structs = PKCS11.constants.select{|c| PKCS11.const_get(c).respond_to?(:ancestors) && !(PKCS11.const_get(c).ancestors & [PKCS11::CStruct, PKCS11::CK_ATTRIBUTE]).empty? }
 
-file 'ext/pk11l_const_def.inc' => 'ext/generate_constants.rb' do
-  sh "#{RbConfig::CONFIG['ruby_install_name']} -I../lib ext/generate_constants.rb --const ext/pk11l_const_def.inc #{HEADER_FILES}"
+  File.write t.name, "PKCS11_STD_STRUCTS = #{std_structs.inspect}"
 end
-file 'ext/pk11l.c' => ['ext/pk11l_struct_def.inc', 'ext/pk11l_struct_impl.inc', 'ext/pk11l_const_def.inc']
 
 task doc_files:  'ext/pk11l_struct.doc'
 

pkcs11_luna/ext/extconf.rb

@@ -4,6 +4,18 @@
 inc, lib = dir_config('luna-dir', '/usr/safenet/lunaclient/samples')
 puts "using Luna Client include:#{inc}"
 
+require_relative "generate_luna_constants"
+require_relative "generate_luna_structs"
+
+header_files = [File.join(inc, "RSA/pkcs11t.h"), File.join(inc, "cryptoki_v2.h")]
+
+args = ["--const", "pk11l_const_def.inc", *header_files]
+puts "running const parser with: #{args.join(" ")}"
+PKCS11::Luna::ConstantParser.run(args)
+
+args = ["--def", "pk11l_struct_def.inc", "--impl", "pk11l_struct_impl.inc", "--doc", "pk11l_struct.doc", *header_files]
+puts "running struct parser with: #{args.join(" ")}"
+PKCS11::Luna::StructParser.run(args)
 
 find_header('pk11_struct_macros.h')
 find_header('pk11_const_macros.h')

pkcs11_luna/ext/generate_constants.rb …cs11_luna/ext/generate_luna_constants.rbpkcs11_luna/ext/generate_constants.rb renamed to pkcs11_luna/ext/generate_luna_constants.rb pkcs11_luna/ext/generate_constants.rb renamed to pkcs11_luna/ext/generate_luna_constants.rb

@@ -2,7 +2,7 @@
 # Quick and dirty parser for PKCS#11 constants and
 # generator for Ruby wrapper classes.
 
-require File.expand_path(File.join(File.dirname(__FILE__), '../../ext/generate_constants'))
+require_relative "generate_constants"
 
 module PKCS11
 module Luna

pkcs11_luna/ext/generate_structs.rb pkcs11_luna/ext/generate_luna_structs.rbpkcs11_luna/ext/generate_structs.rb renamed to pkcs11_luna/ext/generate_luna_structs.rb

@@ -2,9 +2,8 @@
 # Quick and dirty parser for PKCS#11 structs and
 # generator for Ruby wrapper classes.
 
-require 'rubygems'
-require 'pkcs11'
-require File.expand_path(File.join(File.dirname(__FILE__), '../../ext/generate_structs'))
+require_relative "generate_structs"
+require_relative "std_structs"
 
 module PKCS11
 module Luna
@@ -63,7 +62,7 @@ def parse_files(files)
   def start!
     @structs = parse_files(options.files)
     @structs_by_name = @structs.inject({}){|sum, v| sum[v.name]=v; sum }
-    @std_structs_by_name = PKCS11.constants.select{|c| PKCS11.const_get(c).respond_to?(:ancestors) && !(PKCS11.const_get(c).ancestors & [PKCS11::CStruct, PKCS11::CK_ATTRIBUTE]).empty? }.inject({}){|sum, v| sum[v.to_s]=true; sum }
+    @std_structs_by_name = PKCS11_STD_STRUCTS.inject({}){|sum, v| sum[v.to_s]=true; sum }
 
     write_files
   end