Added MLCP functionality

deploy/default.properties

@@ -179,3 +179,9 @@ local-server=localhost
 #dev-server=
 #cert-server=
 #prod-server=
+
+#
+# If you want to use MLCP, make sure mlcp-home is configured correctly. Using a common location as default.
+#
+mlcp-home=/usr/local/mlcp
+mlcp-vmargs=-Xmx512m

deploy/lib/Help.rb

@@ -31,6 +31,7 @@ def self.usage
         corb          Runs Corb against the given environment
         deploy        Loads modules, data, cpf configuration into the given environment
         load          Loads a file or folder into the given environment
+        mlcp          Runs MLCP against the given environment
         recordloader  Runs RecordLoader against the given environment
         test          Runs xquery unit tests against the given environment
         xqsync        Runs XQSync against the given environment
@@ -347,6 +348,26 @@ def self.corb
     DOC
   end
 
+  def self.mlcp
+    <<-DOC.strip_heredoc
+      Usage: ml {env} mlcp [options]
+
+      Runs MLCP with given command-line options agains selected environment.
+      MLCP supports options files natively using the -option_file parameter.
+      The path must a relative or absolute path to a MLCP options file.
+      See http://docs.marklogic.com/guide/ingestion/content-pump#chapter
+
+      General options:
+        -v, [--verbose]  # Verbose output
+        -h, [--help]     # Shows this help
+
+      Roxy applies variable substitution within option files. You may use variables like:
+
+      -input_file_path
+      ${ml.data.dir}/
+    DOC
+  end
+
   def self.plugin
     <<-DOC.strip_heredoc
       Usage: ml {env} plugin [command] [package] [version] [options]

deploy/lib/server_config.rb

@@ -683,6 +683,73 @@ def corb
     end
   end
 
+  def mlcp
+    mlcp_home = @properties['ml.mlcp-home']
+    if @properties['ml.mlcp-home'] == nil || ! File.directory?(File.expand_path(mlcp_home)) || ! File.exists?(File.expand_path("#{mlcp_home}/bin/mlcp.sh"))
+      raise "MLCP not found or mis-configured, please check the mlcp-home setting."
+    end
+
+    # Find all jars required for running MLCP. At least:
+    jars = Dir.glob(ServerConfig.expand_path("#{mlcp_home}/lib/*.jar"))
+    classpath = jars.join(path_separator)
+
+    ARGV.each do |arg|
+      if arg == "-option_file"
+        # remove flag from ARGV
+        index = ARGV.index(arg)
+        ARGV.slice!(index)
+
+        # capture and remove value from ARGV
+        option_file = ARGV[index]
+        ARGV.slice!(index)
+
+        # find and read file if exists
+        option_file = ServerConfig.expand_path("#{@@path}/#{option_file}")
+        if File.exist? option_file
+          logger.debug "Reading options file #{option_file}.."
+          options = File.read option_file
+
+          # substitute properties
+          @properties.sort {|x,y| y <=> x}.each do |k, v|
+            options.gsub!("@#{k}", v)
+          end
+
+          logger.debug "Options after resolving properties:"
+          lines = options.split(/[\n\r]+/).reject! { |line| line.empty? || line.match("^#") }
+
+          lines.each do |line|
+            logger.debug line
+          end
+
+          # and insert the properties back into ARGV
+          ARGV[index,0] = lines
+        else
+          raise "Option file #{option_file} not found."
+        end
+      end
+    end
+
+    if ARGV.length > 0
+      password_prompt
+      connection_string = %Q{ -username #{@properties['ml.user']} -password #{@ml_password} -host #{@properties['ml.server']} -port #{@properties['ml.xcc-port']}}
+
+      args = ARGV.join(" ")
+
+      runme = %Q{java -cp #{classpath} #{@properties['ml.mlcp-vmargs']} com.marklogic.contentpump.ContentPump #{args} #{connection_string}}
+    else
+      runme = %Q{java -cp #{classpath} com.marklogic.contentpump.ContentPump}
+    end
+
+    logger.debug runme
+    logger.info ""
+
+    system runme
+
+    logger.info ""
+
+    ARGV.clear
+  end
+
   def credentials
     logger.info "credentials #{@environment}"
     # ml will error on invalid environment

deploy/lib/xquery/setup.xqy

@@ -1279,7 +1279,7 @@ declare function setup:add-field-includes-R(
         return
           admin:database-included-element(
             $e/db:namespace-uri,
-            $e/db:localname,
+            $e/db:localname/fn:string(.),
             ($e/db:weight, 1.0)[1],
             $e/db:attribute-namespace-uri,
             ($e/db:attribute-localname, "")[1],
@@ -1326,7 +1326,7 @@ declare function setup:add-field-excludes-R(
 
               admin:database-excluded-element(
                 $e/db:namespace-uri,
-                $e/db:localname,
+                $e/db:localname/fn:string(.),
                 ($e/db:attribute-namespace-uri, "")[1],
                 ($e/db:attribute-localname, "")[1],
                 ($e/db:attribute-value, "")[1])',

deploy/sample/build.sample.properties

@@ -142,3 +142,9 @@ local-server=localhost
 #dev-server=
 #cert-server=
 #prod-server=
+
+#
+# If you want to use MLCP, make sure mlcp-home is configured correctly. Using a common location as default.
+#
+mlcp-home=/usr/local/mlcp
+mlcp-vmargs=-Xmx512m

deploy/test/data/ml7-config.xml

@@ -469,7 +469,7 @@
             </included-element>
             <included-element>
               <namespace-uri>http://www.marklogic.com/ns/sample2</namespace-uri>
-              <localname>word-query-include2</localname>
+              <localname>word-query-include2 word-query-include3</localname>
               <weight>1.0</weight>
               <attribute-namespace-uri/>
               <attribute-localname/>
@@ -486,7 +486,7 @@
             </excluded-element>
             <excluded-element>
               <namespace-uri>http://www.marklogic.com/ns/sample2</namespace-uri>
-              <localname>word-query-exclude2</localname>
+              <localname>word-query-exclude2 word-query-exclude3</localname>
               <attribute-namespace-uri/>
               <attribute-localname/>
               <attribute-value/>
@@ -512,7 +512,7 @@
             </included-element>
             <included-element>
               <namespace-uri>http://www.marklogic.com/ns/sample2</namespace-uri>
-              <localname>sample-included-element2</localname>
+              <localname>sample-included-element2 sample-included-element3</localname>
               <weight>1</weight>
               <attribute-namespace-uri/>
               <attribute-localname/>
@@ -529,7 +529,7 @@
             </excluded-element>
             <excluded-element>
               <namespace-uri>http://www.marklogic.com/ns/sample2</namespace-uri>
-              <localname>sample-excluded-element2</localname>
+              <localname>sample-excluded-element2 sample-excluded-element3</localname>
               <attribute-namespace-uri/>
               <attribute-localname/>
               <attribute-value/>
@@ -544,15 +544,15 @@
           <included-elements>
             <included-element>
               <namespace-uri>http://www.marklogic.com/ns/sample</namespace-uri>
-              <localname>sample-included-element</localname>
+              <localname>sample2-included-element</localname>
               <weight>1</weight>
               <attribute-namespace-uri/>
               <attribute-localname/>
               <attribute-value/>
             </included-element>
             <included-element>
               <namespace-uri>http://www.marklogic.com/ns/sample2</namespace-uri>
-              <localname>sample-included-element2</localname>
+              <localname>sample2-included-element2 sample2-included-element3</localname>
               <weight>1</weight>
               <attribute-namespace-uri/>
               <attribute-localname/>
@@ -562,14 +562,14 @@
           <excluded-elements>
             <excluded-element>
               <namespace-uri>http://www.marklogic.com/ns/sample</namespace-uri>
-              <localname>sample-excluded-element</localname>
+              <localname>sample2-excluded-element</localname>
               <attribute-namespace-uri/>
               <attribute-localname/>
               <attribute-value/>
             </excluded-element>
             <excluded-element>
               <namespace-uri>http://www.marklogic.com/ns/sample2</namespace-uri>
-              <localname>sample-excluded-element2</localname>
+              <localname>sample2-excluded-element2 sample2-excluded-element3</localname>
               <attribute-namespace-uri/>
               <attribute-localname/>
               <attribute-value/>