From 27b383bdc0fc433fda2171f4ecbb9394a6517adf Mon Sep 17 00:00:00 2001 From: liangadam <45301969+liangadam@users.noreply.github.com> Date: Fri, 12 Nov 2021 01:02:19 +0800 Subject: [PATCH 01/17] Improved activation function. --- deepmd/utils/tabulate.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/deepmd/utils/tabulate.py b/deepmd/utils/tabulate.py index f0dc571142..1e2cdcd40c 100644 --- a/deepmd/utils/tabulate.py +++ b/deepmd/utils/tabulate.py @@ -68,6 +68,14 @@ def __init__(self, self.functype = 1 elif activation_fn == ACTIVATION_FN_DICT["gelu"]: self.functype = 2 + elif activation_fn == ACTIVATION_FN_DICT["relu"]: + self.functype = 3 + elif activation_fn == ACTIVATION_FN_DICT["relu6"]: + self.functype = 4 + elif activation_fn == ACTIVATION_FN_DICT["softplus"]: + self.functype = 5 + elif activation_fn == ACTIVATION_FN_DICT["sigmoid"]: + self.functype = 6 else: raise RuntimeError("Unknown actication function type!") self.activation_fn = activation_fn From c8cb970ca2649b252d9022f0d001b9c54a93667d Mon Sep 17 00:00:00 2001 From: liangadam <45301969+liangadam@users.noreply.github.com> Date: Fri, 12 Nov 2021 01:03:13 +0800 Subject: [PATCH 02/17] Improved activation function. --- source/op/unaggregated_grad.cc | 46 ++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/source/op/unaggregated_grad.cc b/source/op/unaggregated_grad.cc index 89c14a84fb..c5f872ab42 100644 --- a/source/op/unaggregated_grad.cc +++ b/source/op/unaggregated_grad.cc @@ -52,6 +52,36 @@ FPTYPE grad(const FPTYPE xbar, const FPTYPE y, const int functype) //functype=t const FPTYPE var = tanh(SQRT_2_PI * (xbar + GGELU * xbar * xbar * xbar)); return 0.5 * SQRT_2_PI * xbar * (1 - var * var) * (3 * GGELU * xbar * xbar + 1) + 0.5 * var + 0.5; } + case 3: + { + if(xbar<=0) + { + return 0; + } + else + { + return 1; + } + } + case 4: + { + if(xbar<=0 || xbar>=6) + { + return 0; + } + else + { + return 1; + } + } + case 5: + { + return 1.0-1.0/(1.0+exp(xbar)); + } + case 6: + { + return y*(1-y); + } default: return -1; } @@ -71,6 +101,22 @@ FPTYPE grad_grad(const FPTYPE xbar, const FPTYPE y, const int functype) const FPTYPE var2 = SQRT_2_PI * (1 - var1 * var1) * (3 * GGELU * xbar * xbar + 1); return 3 * GGELU * SQRT_2_PI * xbar * xbar * (1 - var1 * var1) - SQRT_2_PI * xbar * var2 * (3 * GGELU * xbar * xbar + 1) * var1 + var2; } + case 3: + { + return 0; + } + case 4: + { + return 0; + } + case 5: + { + return exp(xbar)/((1+exp(xbar))*(1+exp(xbar))); + } + case 6: + { + return y*(1-y)*(1-2*y); + } default: return -1; } From 38b8c9bb1ff15df5846d703733a0f2e51427a192 Mon Sep 17 00:00:00 2001 From: liangadam <45301969+liangadam@users.noreply.github.com> Date: Fri, 12 Nov 2021 01:15:49 +0800 Subject: [PATCH 03/17] Added description of available activation functions. --- doc/train/train-input.rst | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/doc/train/train-input.rst b/doc/train/train-input.rst index 2e3fe83d4b..d00fea04ab 100644 --- a/doc/train/train-input.rst +++ b/doc/train/train-input.rst @@ -2,5 +2,17 @@ Training Parameters ====================================== .. note:: One can load, modify, and export the input file by using our effective web-based tool `DP-GUI `_. All training parameters below can be set in DP-GUI. By clicking "SAVE JSON", one can download the input file for furthur training. + + + Available activation functions for descriptor: + tanh + gelu + relu + relu6 + softplus + sigmoid + You can specify one of the above functions as the activation function in the descriptor's configuration in "input.json". + For example: + "activation_function": "sigmoid" .. include:: ../train-input-auto.rst From 94923cda73cb649a2ac0b9bfd665d8a9a2626ea7 Mon Sep 17 00:00:00 2001 From: liangadam <45301969+liangadam@users.noreply.github.com> Date: Fri, 12 Nov 2021 01:20:02 +0800 Subject: [PATCH 04/17] Added description of available activation functions. --- doc/train/train-input.rst | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/doc/train/train-input.rst b/doc/train/train-input.rst index d00fea04ab..abc9f77c5f 100644 --- a/doc/train/train-input.rst +++ b/doc/train/train-input.rst @@ -4,13 +4,13 @@ Training Parameters One can load, modify, and export the input file by using our effective web-based tool `DP-GUI `_. All training parameters below can be set in DP-GUI. By clicking "SAVE JSON", one can download the input file for furthur training. - Available activation functions for descriptor: - tanh - gelu - relu - relu6 - softplus - sigmoid + **Available activation functions for descriptor:** + -tanh + -gelu + -relu + -relu6 + -softplus + -sigmoid You can specify one of the above functions as the activation function in the descriptor's configuration in "input.json". For example: "activation_function": "sigmoid" From 5eed2a0fbea1a36ac4eb6e48d53fc9cb177dca2b Mon Sep 17 00:00:00 2001 From: liangadam <45301969+liangadam@users.noreply.github.com> Date: Fri, 12 Nov 2021 01:21:37 +0800 Subject: [PATCH 05/17] Added description of available activation functions. --- doc/train/train-input.rst | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/doc/train/train-input.rst b/doc/train/train-input.rst index abc9f77c5f..466418fe68 100644 --- a/doc/train/train-input.rst +++ b/doc/train/train-input.rst @@ -5,12 +5,12 @@ Training Parameters **Available activation functions for descriptor:** - -tanh - -gelu - -relu - -relu6 - -softplus - -sigmoid + -tanh + -gelu + -relu + -relu6 + -softplus + -sigmoid You can specify one of the above functions as the activation function in the descriptor's configuration in "input.json". For example: "activation_function": "sigmoid" From 5bbab945fb6ee98678d544066b6a6cdba5d2e1b9 Mon Sep 17 00:00:00 2001 From: liangadam <45301969+liangadam@users.noreply.github.com> Date: Fri, 12 Nov 2021 01:22:12 +0800 Subject: [PATCH 06/17] Added description of available activation functions. --- doc/train/train-input.rst | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/doc/train/train-input.rst b/doc/train/train-input.rst index 466418fe68..51f58de309 100644 --- a/doc/train/train-input.rst +++ b/doc/train/train-input.rst @@ -5,12 +5,12 @@ Training Parameters **Available activation functions for descriptor:** - -tanh - -gelu - -relu - -relu6 - -softplus - -sigmoid + tanh + gelu + relu + relu6 + softplus + sigmoid You can specify one of the above functions as the activation function in the descriptor's configuration in "input.json". For example: "activation_function": "sigmoid" From 5f8bd2e02da5f775378d228a32f225d0ba849dbf Mon Sep 17 00:00:00 2001 From: liangadam <45301969+liangadam@users.noreply.github.com> Date: Fri, 12 Nov 2021 01:23:47 +0800 Subject: [PATCH 07/17] Update train-input.rst --- doc/train/train-input.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/train/train-input.rst b/doc/train/train-input.rst index 51f58de309..29be297041 100644 --- a/doc/train/train-input.rst +++ b/doc/train/train-input.rst @@ -5,7 +5,9 @@ Training Parameters **Available activation functions for descriptor:** + tanh + gelu relu relu6 From 8a70736a56e7f88630add59a63caaf83beb20925 Mon Sep 17 00:00:00 2001 From: liangadam <45301969+liangadam@users.noreply.github.com> Date: Fri, 12 Nov 2021 01:25:02 +0800 Subject: [PATCH 08/17] Update train-input.rst --- doc/train/train-input.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/train/train-input.rst b/doc/train/train-input.rst index 29be297041..27f7f505bb 100644 --- a/doc/train/train-input.rst +++ b/doc/train/train-input.rst @@ -6,10 +6,10 @@ Training Parameters **Available activation functions for descriptor:** - tanh + -tanh - gelu - relu + -gelu +-relu relu6 softplus sigmoid From 77cf6b67b5a313247417696d4c582c0fdfb6d461 Mon Sep 17 00:00:00 2001 From: liangadam <45301969+liangadam@users.noreply.github.com> Date: Fri, 12 Nov 2021 01:25:41 +0800 Subject: [PATCH 09/17] Update train-input.rst --- doc/train/train-input.rst | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/doc/train/train-input.rst b/doc/train/train-input.rst index 27f7f505bb..a109c4dc74 100644 --- a/doc/train/train-input.rst +++ b/doc/train/train-input.rst @@ -6,9 +6,8 @@ Training Parameters **Available activation functions for descriptor:** - -tanh - - -gelu +-tanh +-gelu -relu relu6 softplus From 7cda479f109174b1d911bf4d95aba28a55f23269 Mon Sep 17 00:00:00 2001 From: liangadam <45301969+liangadam@users.noreply.github.com> Date: Fri, 12 Nov 2021 01:27:11 +0800 Subject: [PATCH 10/17] Added description of available activation functions. --- doc/train/train-input.rst | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/doc/train/train-input.rst b/doc/train/train-input.rst index a109c4dc74..17a1c2f449 100644 --- a/doc/train/train-input.rst +++ b/doc/train/train-input.rst @@ -4,14 +4,13 @@ Training Parameters One can load, modify, and export the input file by using our effective web-based tool `DP-GUI `_. All training parameters below can be set in DP-GUI. By clicking "SAVE JSON", one can download the input file for furthur training. - **Available activation functions for descriptor:** - --tanh --gelu --relu - relu6 - softplus - sigmoid + **Available activation functions for descriptor:** + - tanh + - gelu + - relu + - relu6 + - softplus + - sigmoid You can specify one of the above functions as the activation function in the descriptor's configuration in "input.json". For example: "activation_function": "sigmoid" From 422863a55221ba399904d9e2612733943df40786 Mon Sep 17 00:00:00 2001 From: liangadam <45301969+liangadam@users.noreply.github.com> Date: Fri, 12 Nov 2021 01:27:44 +0800 Subject: [PATCH 11/17] Added description of available activation functions. --- doc/train/train-input.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/train/train-input.rst b/doc/train/train-input.rst index 17a1c2f449..fcd1825ec2 100644 --- a/doc/train/train-input.rst +++ b/doc/train/train-input.rst @@ -4,7 +4,7 @@ Training Parameters One can load, modify, and export the input file by using our effective web-based tool `DP-GUI `_. All training parameters below can be set in DP-GUI. By clicking "SAVE JSON", one can download the input file for furthur training. - **Available activation functions for descriptor:** + **Available activation functions for descriptor**: - tanh - gelu - relu From f28e9d180e36f219e29c70e5e08da90096d06818 Mon Sep 17 00:00:00 2001 From: liangadam <45301969+liangadam@users.noreply.github.com> Date: Fri, 12 Nov 2021 01:28:02 +0800 Subject: [PATCH 12/17] Update train-input.rst --- doc/train/train-input.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/train/train-input.rst b/doc/train/train-input.rst index fcd1825ec2..b1b94f86f0 100644 --- a/doc/train/train-input.rst +++ b/doc/train/train-input.rst @@ -4,8 +4,8 @@ Training Parameters One can load, modify, and export the input file by using our effective web-based tool `DP-GUI `_. All training parameters below can be set in DP-GUI. By clicking "SAVE JSON", one can download the input file for furthur training. - **Available activation functions for descriptor**: - - tanh + **Available activation functions for descriptor:** +- tanh - gelu - relu - relu6 From c8f546e0db02a120c9612c051df6f1e5994999bd Mon Sep 17 00:00:00 2001 From: liangadam <45301969+liangadam@users.noreply.github.com> Date: Fri, 12 Nov 2021 01:28:23 +0800 Subject: [PATCH 13/17] Added description of available activation functions. --- doc/train/train-input.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/train/train-input.rst b/doc/train/train-input.rst index b1b94f86f0..8f1bbb9ced 100644 --- a/doc/train/train-input.rst +++ b/doc/train/train-input.rst @@ -6,11 +6,11 @@ Training Parameters **Available activation functions for descriptor:** - tanh - - gelu - - relu - - relu6 - - softplus - - sigmoid +- gelu +- relu +- relu6 +- softplus +- sigmoid You can specify one of the above functions as the activation function in the descriptor's configuration in "input.json". For example: "activation_function": "sigmoid" From 7d566f27da5de58c52eb0f140ea92350eccfe973 Mon Sep 17 00:00:00 2001 From: liangadam <45301969+liangadam@users.noreply.github.com> Date: Fri, 12 Nov 2021 01:28:56 +0800 Subject: [PATCH 14/17] Added description of available activation functions. --- doc/train/train-input.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/train/train-input.rst b/doc/train/train-input.rst index 8f1bbb9ced..3eddcbc20d 100644 --- a/doc/train/train-input.rst +++ b/doc/train/train-input.rst @@ -11,6 +11,7 @@ Training Parameters - relu6 - softplus - sigmoid + You can specify one of the above functions as the activation function in the descriptor's configuration in "input.json". For example: "activation_function": "sigmoid" From 4a928c1ad1c67e35183b6bec7a53779381f3617a Mon Sep 17 00:00:00 2001 From: liangadam <45301969+liangadam@users.noreply.github.com> Date: Fri, 12 Nov 2021 01:29:30 +0800 Subject: [PATCH 15/17] Added description of available activation functions. --- doc/train/train-input.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/train/train-input.rst b/doc/train/train-input.rst index 3eddcbc20d..c5f69dd94a 100644 --- a/doc/train/train-input.rst +++ b/doc/train/train-input.rst @@ -12,8 +12,8 @@ Training Parameters - softplus - sigmoid - You can specify one of the above functions as the activation function in the descriptor's configuration in "input.json". - For example: +You can specify one of the above functions as the activation function in the descriptor's configuration in "input.json". +For example: "activation_function": "sigmoid" .. include:: ../train-input-auto.rst From 94382a0536e57bbd6ad4ad199d89a4e23de432fa Mon Sep 17 00:00:00 2001 From: liangadam <45301969+liangadam@users.noreply.github.com> Date: Fri, 12 Nov 2021 01:32:07 +0800 Subject: [PATCH 16/17] Added description of available activation functions. --- doc/freeze/compress.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/doc/freeze/compress.md b/doc/freeze/compress.md index ed55db0c4b..362ae22840 100644 --- a/doc/freeze/compress.md +++ b/doc/freeze/compress.md @@ -83,3 +83,14 @@ The model compression interface requires the version of deepmd-kit used in origi **Acceptable descriptor type** Note only descriptors with `se_e2_a` or `se_e3` type are supported by the model compression feature. Hybrid mixed with above descriptors is also supported. + + +**Available activation functions for descriptor:** +- tanh +- gelu +- relu +- relu6 +- softplus +- sigmoid + + From 6d98a3105f3fd99bc6a69b71202c43fafce7de9f Mon Sep 17 00:00:00 2001 From: liangadam <45301969+liangadam@users.noreply.github.com> Date: Sun, 14 Nov 2021 14:57:29 +0800 Subject: [PATCH 17/17] Update train-input.rst --- doc/train/train-input.rst | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/doc/train/train-input.rst b/doc/train/train-input.rst index c5f69dd94a..2e3fe83d4b 100644 --- a/doc/train/train-input.rst +++ b/doc/train/train-input.rst @@ -2,18 +2,5 @@ Training Parameters ====================================== .. note:: One can load, modify, and export the input file by using our effective web-based tool `DP-GUI `_. All training parameters below can be set in DP-GUI. By clicking "SAVE JSON", one can download the input file for furthur training. - - - **Available activation functions for descriptor:** -- tanh -- gelu -- relu -- relu6 -- softplus -- sigmoid - -You can specify one of the above functions as the activation function in the descriptor's configuration in "input.json". -For example: - "activation_function": "sigmoid" .. include:: ../train-input-auto.rst