Added Operand<TFloat32> learningRateOperand test case for learning rate.

tensorflow-framework/src/test/java/org/tensorflow/framework/optimizers/AdaDeltaTest.java

@@ -167,6 +167,109 @@ public void testBasic() {
     }
   }
 
+  @Test
+  public void testBasicWithLROperand() {
+    int numUpdates = 4; // # number of ADADELTA steps to perform
+    float[] grads = {0.2F, 0.1F, 0.01F};
+    float[] lrs = {1.0F, 0.5F, 0.1F};
+
+    float rho = 0.95F;
+    float epsilon = 1e-8F;
+
+    for (float grad : grads) {
+      for (float lr : lrs) {
+        try (TestSession session = TestSession.createTestSession(tfMode)) {
+          Ops tf = session.getTF();
+          // this just uses a trivial operand
+          try (AdaDelta instance =
+              new AdaDelta(
+                  session.getGraph(),
+                  tf.math.mul(tf.constant(lr), tf.constant(1.f)),
+                  rho,
+                  epsilon)) {
+
+            float[] var0Init = {1.0F, 2.0F};
+            float[] var1Init = {3.0F, 4.0F};
+            float[] fgrads = {grad, grad};
+            Shape shape = Shape.of(var0Init.length);
+            Variable<TFloat32> var0 = tf.withName("var0").variable(shape, TFloat32.DTYPE);
+            Variable<TFloat32> var1 = tf.withName("var1").variable(shape, TFloat32.DTYPE);
+
+            Assign<TFloat32> var0Initializer = tf.assign(var0, tf.constant(var0Init));
+            Assign<TFloat32> var1Initializer = tf.assign(var1, tf.constant(var1Init));
+
+            Constant<TFloat32> cgrads = tf.constant(fgrads);
+            float accum = 0.0F;
+            float accumUpdate = 0.0F;
+
+            /* build the GradsAnvVars */
+            List<GradAndVar<? extends TType>> gradsAndVars = new ArrayList<>();
+            gradsAndVars.add(new GradAndVar<>(cgrads.asOutput(), var0.asOutput()));
+            gradsAndVars.add(new GradAndVar<>(cgrads.asOutput(), var1.asOutput()));
+
+            /*apply gradients */
+            Op adadeltaUpdate = instance.applyGradients(gradsAndVars, "AdaDeltaTest");
+
+            /* Create and validate the shapes of the slota */
+            @SuppressWarnings("unchecked")
+            Variable<TFloat32>[] slots = new Variable[2];
+            @SuppressWarnings("unchecked")
+            Variable<TFloat32>[] slotUpdates = new Variable[2];
+
+            slots[0] = instance.getSlot(var0.asOutput(), ACCUMULATOR).get();
+            assertEquals(slots[0].asOutput().shape(), var0.asOutput().shape());
+
+            slotUpdates[0] = instance.getSlot(var0.asOutput(), ACCUMULATOR_UPDATE).get();
+            assertEquals(slotUpdates[0].asOutput().shape(), var0.asOutput().shape());
+
+            slots[1] = instance.getSlot(var1.asOutput(), ACCUMULATOR).get();
+            assertEquals(slots[1].asOutput().shape(), var1.asOutput().shape());
+
+            slotUpdates[1] = instance.getSlot(var1.asOutput(), ACCUMULATOR_UPDATE).get();
+            assertEquals(slotUpdates[1].asOutput().shape(), var1.asOutput().shape());
+
+            /* initialize the local variables */
+            session.run(var0Initializer);
+            session.run(var1Initializer);
+
+            /* initialize the accumulators */
+            session.run(tf.init());
+
+            /* make sure the variables were initialized properly */
+            session.evaluate(var0Init, var0);
+            session.evaluate(var1Init, var1);
+
+            float[] updates = new float[numUpdates];
+            float totUpdate = 0;
+            for (int step = 0; step < numUpdates; step++) {
+
+              session.run(adadeltaUpdate, instance.getFeedMap());
+              accum = accum * rho + (float) Math.pow(grad, 2) * (1.0F - rho);
+              updates[step] =
+                  ((float) Math.sqrt(accumUpdate + epsilon)
+                      * (float) (1 / Math.sqrt(accum + epsilon))
+                      * grad);
+              accumUpdate =
+                  (accumUpdate * rho + ((float) Math.pow(updates[step], 2) * (1.0F - rho)));
+              totUpdate += updates[step] * lr;
+
+              for (int i = 0; i < 2; i++) {
+                session.evaluate(accum, slots[i]);
+                session.evaluate(accumUpdate, slotUpdates[i]);
+              }
+
+              Float[] var0InitUpdate = {var0Init[0] - totUpdate, var0Init[1] - totUpdate};
+              Float[] var1InitUpdate = {var1Init[0] - totUpdate, var1Init[1] - totUpdate};
+
+              session.evaluate(var0InitUpdate, var0);
+              session.evaluate(var1InitUpdate, var1);
+            }
+          }
+        }
+      }
+    }
+  }
+
   @Test
   public void testWithLearningRateDecay() {
     int numSteps = 4; // # number of ADADELTA steps to perform
@@ -224,10 +327,10 @@ public void testWithLearningRateDecay() {
         session.run(var0Initializer);
         session.run(var1Initializer);
 
-        /** initialize the accumulators */
+        /* initialize the accumulators */
         session.run(tf.init());
 
-        /** make sure the variables were initialized properly */
+        /* make sure the variables were initialized properly */
         session.evaluate(var0Init, var0);
         session.evaluate(var1Init, var1);
 

tensorflow-framework/src/test/java/org/tensorflow/framework/optimizers/AdaGradDATest.java

@@ -97,14 +97,63 @@ public void testBasic() {
     }
   }
 
+  @Test
+  public void testBasicWithLROperand() {
+    float[] var0Init = {0.0F, 0.0F};
+    float[] var1Init = {0.0F, 0.0F};
+    float[] grads0Init = {0.1F, 0.2F};
+    float[] grads1Init = {0.01F, 0.02F};
+    float learningRate = 1.5F;
+    try (TestSession session = TestSession.createTestSession(tfMode)) {
+      Ops tf = session.getTF();
+      try (AdaGradDA instance =
+          new AdaGradDA(
+              session.getGraph(), tf.math.mul(tf.constant(learningRate), tf.constant(2.f)))) {
+
+        Shape shape0 = Shape.of(var0Init.length);
+        Shape shape1 = Shape.of(var1Init.length);
+        Variable<TFloat32> var0 = tf.withName("var0").variable(shape0, TFloat32.DTYPE);
+        Variable<TFloat32> var1 = tf.withName("var1").variable(shape1, TFloat32.DTYPE);
+
+        Assign<TFloat32> var0Initializer = tf.assign(var0, tf.constant(var0Init));
+        Assign<TFloat32> var1Initializer = tf.assign(var1, tf.constant(var1Init));
+
+        Constant<TFloat32> grads0 = tf.constant(grads0Init);
+        Constant<TFloat32> grads1 = tf.constant(grads1Init);
+
+        /* initialize the local variables */
+
+        session.run(var0Initializer);
+        session.run(var1Initializer);
+
+        /* build the GradsAnvVars */
+        List<Optimizer.GradAndVar<? extends TType>> gradsAndVars = new ArrayList<>();
+        gradsAndVars.add(new Optimizer.GradAndVar<>(grads0.asOutput(), var0.asOutput()));
+        gradsAndVars.add(new Optimizer.GradAndVar<>(grads1.asOutput(), var1.asOutput()));
+
+        Op adaUpdate = instance.applyGradients(gradsAndVars, "AdGradDATest");
+
+        /* initialize the accumulators */
+        session.run(tf.init());
+
+        session.evaluate(var0Init, var0);
+        session.evaluate(var1Init, var1);
+        session.run(adaUpdate, instance.getFeedMap());
+        float[] expected0 = {-0.904534F, -1.603567F};
+        session.evaluate(expected0, var0);
+        float[] expected1 = {-0.094821f, -0.189358f};
+        session.evaluate(expected1, var1);
+      }
+    }
+  }
+
   @Test
   public void testWithLearningRateDecay() {
     float[] var0Init = {0.0F, 0.0F};
     float[] var1Init = {0.0F, 0.0F};
     float[] grads0Init = {0.1F, 0.2F};
     float[] grads1Init = {0.01F, 0.02F};
     float epsilon = 1e-8F;
-    float epsilon1 = 1e-5F;
     int numSteps = 4;
     float learningRate = 3.0F;
     try (TestSession session = TestSession.createTestSession(tfMode);

tensorflow-framework/src/test/java/org/tensorflow/framework/optimizers/AdaGradTest.java

@@ -111,6 +111,70 @@ public void testBasic() {
     }
   }
 
+  @Test
+  public void testBasicWithLROperand() {
+    int numSteps = 3;
+    float[] var0Init = {1.0F, 2.0F};
+    float[] var1Init = {3.0F, 4.0F};
+    float[] grads0Init = {0.1F, 0.1F};
+    float[] grads1Init = {0.01F, 0.01F};
+
+    float learningRate = 1.0F;
+
+    try (TestSession session = TestSession.createTestSession(tfMode)) {
+      Ops tf = session.getTF();
+      try (AdaGrad instance =
+          new AdaGrad(
+              session.getGraph(), tf.math.mul(tf.constant(learningRate), tf.constant(3.f)), 0.1f)) {
+
+        Shape shape0 = Shape.of(var0Init.length);
+        Shape shape1 = Shape.of(var1Init.length);
+        Variable<TFloat32> var0 = tf.withName("var0").variable(shape0, TFloat32.DTYPE);
+        Variable<TFloat32> var1 = tf.withName("var1").variable(shape1, TFloat32.DTYPE);
+
+        Assign<TFloat32> var0Initializer = tf.assign(var0, tf.constant(var0Init));
+        Assign<TFloat32> var1Initializer = tf.assign(var1, tf.constant(var1Init));
+
+        Constant<TFloat32> grads0 = tf.constant(grads0Init);
+        Constant<TFloat32> grads1 = tf.constant(grads1Init);
+
+        /* build the GradsAnvVars */
+        List<Optimizer.GradAndVar<? extends TType>> gradsAndVars = new ArrayList<>();
+        gradsAndVars.add(new Optimizer.GradAndVar<>(grads0.asOutput(), var0.asOutput()));
+        gradsAndVars.add(new Optimizer.GradAndVar<>(grads1.asOutput(), var1.asOutput()));
+
+        Op adaUpdate = instance.applyGradients(gradsAndVars, "AdGradTest");
+
+        @SuppressWarnings("unchecked")
+        Variable<TFloat32>[] accumulatorSlots = new Variable[2];
+        accumulatorSlots[0] = instance.getSlot(var0.asOutput(), ACCUMULATOR).get();
+        assertEquals(accumulatorSlots[0].asOutput().shape(), var0.asOutput().shape());
+
+        accumulatorSlots[1] = instance.getSlot(var1.asOutput(), ACCUMULATOR).get();
+        assertEquals(accumulatorSlots[1].asOutput().shape(), var1.asOutput().shape());
+
+        /* initialize the local variables */
+        session.run(var0Initializer);
+        session.run(var1Initializer);
+
+        /* initialize the accumulators */
+        session.run(tf.init());
+
+        /* make sure the variables were initialized properly */
+        session.evaluate(var0Init, var0);
+        session.evaluate(var1Init, var1);
+
+        for (int step = 0; step < numSteps; step++) {
+          session.run(adaUpdate, instance.getFeedMap());
+        }
+        float[] expected0 = {-1.6026098728179932f, -0.6026098728179932f};
+        session.evaluate(expected0, var0);
+        float[] expected1 = {2.715679168701172f, 3.715679168701172f};
+        session.evaluate(expected1, var1);
+      }
+    }
+  }
+
   @Test
   public void testWithLearningRateDecay() {
     int numSteps = 3;