From 5d26f17019dff096cc12629228f02249eb64d40f Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Tue, 23 Aug 2022 21:43:48 -0400 Subject: [PATCH 01/21] remove white space from train_attr/training_script (#1870) #921 discussed that the tensors are compressed in the graph file. But it looks no... So at least we remove white space. --- deepmd/entrypoints/train.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/deepmd/entrypoints/train.py b/deepmd/entrypoints/train.py index 4a67eee9d4..edc76b4d85 100755 --- a/deepmd/entrypoints/train.py +++ b/deepmd/entrypoints/train.py @@ -97,7 +97,8 @@ def train( json.dump(jdata, fp, indent=4) # save the training script into the graph - tf.constant(json.dumps(jdata), name='train_attr/training_script', dtype=tf.string) + # remove white spaces as it is not compressed + tf.constant(json.dumps(jdata, separators=(',', ':')), name='train_attr/training_script', dtype=tf.string) for message in WELCOME + CITATION + BUILD: log.info(message) From 3ffcd4b2088b0daa59b044a68eba573738550b1d Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Tue, 23 Aug 2022 21:45:07 -0400 Subject: [PATCH 02/21] replace FastGFile with GFile (#1874) FastGFile throws a deprecated warning. See https://github.com/tensorflow/tensorflow/blob/d8ce9f9c301d021a69953134185ab728c1c248d3/tensorflow/python/platform/gfile.py#L128 Signed-off-by: Jinzhe Zeng Signed-off-by: Jinzhe Zeng --- deepmd/utils/convert.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/deepmd/utils/convert.py b/deepmd/utils/convert.py index 7dc8ebb06f..bbe9e067ef 100644 --- a/deepmd/utils/convert.py +++ b/deepmd/utils/convert.py @@ -1,7 +1,6 @@ import os from deepmd.env import tf from google.protobuf import text_format -from tensorflow.python.platform import gfile def convert_13_to_21(input_model: str, output_model: str): @@ -132,7 +131,7 @@ def convert_pb_to_pbtxt(pbfile: str, pbtxtfile: str): pbtxtfile : str filename of the output graph text """ - with gfile.FastGFile(pbfile, 'rb') as f: + with tf.gfile.GFile(pbfile, 'rb') as f: graph_def = tf.GraphDef() graph_def.ParseFromString(f.read()) tf.import_graph_def(graph_def, name='') @@ -148,7 +147,7 @@ def convert_pbtxt_to_pb(pbtxtfile: str, pbfile: str): pbfile : str filename of the output graph """ - with tf.gfile.FastGFile(pbtxtfile, 'r') as f: + with tf.gfile.GFile(pbtxtfile, 'r') as f: graph_def = tf.GraphDef() file_content = f.read() # Merges the human-readable string in `file_content` into `graph_def`. @@ -331,4 +330,4 @@ def convert_dp20_to_dp21(fname: str): .replace('TabulateFusionGrad', 'TabulateFusionSeAGrad')\ .replace('TabulateFusionGradGrad', 'TabulateFusionSeAGradGrad') with open(fname, 'w') as fp: - fp.write(file_content) \ No newline at end of file + fp.write(file_content) From 3b00375dd5f1468ab1b462af49f96a7c56979272 Mon Sep 17 00:00:00 2001 From: Duo <50307526+iProzd@users.noreply.github.com> Date: Fri, 26 Aug 2022 10:49:20 +0800 Subject: [PATCH 03/21] Upload attention based model (#1866) Here I upload the implementation of attention based model (DPA-1), see details in paper: https://arxiv.org/abs/2208.08236. The changes can mainly be listed as following: 1. Add a new descriptor '**se_atten**', using attention schemes and type embeddings (deepmd/descriptor/se_atten.py); 2. Add a new custom op '**ProdEnvMatAMix**', mixing atoms with different types in the sorting, which is only suitable for descriptor using type embeddings (source/op/prod_env_mat_multi_device.cc and other sub files, here I try the best to reuse the previous codes); 3. Other changes to match the newly added descriptor, such as 'sel' supporting 'int' type as input (deepmd/entrypoints/train.py ) and type embedding with padding (deepmd/utils/type_embed.py); 4. **Mixed_type**: a new format of DeepmdData to put different systems sharing the same 'nloc' (instead of same fingerprints) together. (deepmd/utils/data.py & data_system.py) Note that this is only the training code on this new format of data. The generation code of this new format will soon be uploaded in _dpdata_. 5. An example of DPA-1 on water system (examples/water/se_atten/input.json); 6. The unittests on both C++&CUDA&ROCM and python interface of newly added apis(source/lib/tests & source/tests). Unittests passed individually for these newly added features. Thanks for reviewing and disccusions are welcome. --- README.md | 1 + deepmd/common.py | 6 +- deepmd/descriptor/__init__.py | 1 + deepmd/descriptor/se_atten.py | 777 ++++++++++++++ deepmd/entrypoints/train.py | 5 +- deepmd/fit/ener.py | 53 +- deepmd/model/ener.py | 41 +- deepmd/train/trainer.py | 30 +- deepmd/utils/argcheck.py | 62 +- deepmd/utils/data.py | 26 +- deepmd/utils/data_system.py | 11 + deepmd/utils/network.py | 5 +- deepmd/utils/type_embed.py | 15 +- doc/data/data-conv.md | 2 + doc/data/system.md | 2 +- doc/images/model_se_atten.png | Bin 0 -> 99657 bytes doc/model/index.md | 1 + doc/model/index.rst | 1 + doc/model/train-se-atten.md | 120 +++ examples/water/se_atten/input.json | 70 ++ source/lib/include/neighbor_list.h | 33 + source/lib/include/prod_env_mat.h | 9 +- source/lib/src/cuda/neighbor_list.cu | 78 ++ source/lib/src/cuda/prod_env_mat.cu | 12 +- source/lib/src/neighbor_list.cc | 49 + source/lib/src/prod_env_mat.cc | 27 +- source/lib/src/rocm/neighbor_list.hip.cu | 78 ++ source/lib/src/rocm/prod_env_mat.hip.cu | 12 +- source/lib/tests/test_env_mat_a_mix.cc | 1055 +++++++++++++++++++ source/op/prod_env_mat_multi_device.cc | 427 +++++++- source/tests/common.py | 45 +- source/tests/test_data_large_batch.py | 173 +++ source/tests/test_descrpt_se_atten.py | 252 +++++ source/tests/test_examples.py | 1 + source/tests/test_fitting_ener_type.py | 9 +- source/tests/test_model_se_atten.py | 138 +++ source/tests/water_se_atten.json | 66 ++ source/tests/water_se_atten_mixed_type.json | 66 ++ 38 files changed, 3672 insertions(+), 87 deletions(-) create mode 100644 deepmd/descriptor/se_atten.py create mode 100644 doc/images/model_se_atten.png create mode 100644 doc/model/train-se-atten.md create mode 100644 examples/water/se_atten/input.json create mode 100644 source/lib/tests/test_env_mat_a_mix.cc create mode 100644 source/tests/test_data_large_batch.py create mode 100644 source/tests/test_descrpt_se_atten.py create mode 100644 source/tests/test_model_se_atten.py create mode 100644 source/tests/water_se_atten.json create mode 100644 source/tests/water_se_atten_mixed_type.json diff --git a/README.md b/README.md index 357ff43fba..6f72f21c14 100644 --- a/README.md +++ b/README.md @@ -94,6 +94,7 @@ A full [document](doc/train/train-input-auto.rst) on options in the training inp - [Descriptor `"se_e2_a"`](doc/model/train-se-e2-a.md) - [Descriptor `"se_e2_r"`](doc/model/train-se-e2-r.md) - [Descriptor `"se_e3"`](doc/model/train-se-e3.md) + - [Descriptor `"se_atten"`](doc/model/train-se-atten.md) - [Descriptor `"hybrid"`](doc/model/train-hybrid.md) - [Descriptor `sel`](doc/model/sel.md) - [Fit energy](doc/model/train-energy.md) diff --git a/deepmd/common.py b/deepmd/common.py index 1146f291d5..6129011bbc 100644 --- a/deepmd/common.py +++ b/deepmd/common.py @@ -405,8 +405,8 @@ def j_loader(filename: Union[str, Path]) -> Dict[str, Any]: def get_activation_func( - activation_fn: "_ACTIVATION", -) -> Callable[[tf.Tensor], tf.Tensor]: + activation_fn: Union["_ACTIVATION", None], +) -> Union[Callable[[tf.Tensor], tf.Tensor], None]: """Get activation function callable based on string name. Parameters @@ -424,6 +424,8 @@ def get_activation_func( RuntimeError if unknown activation function is specified """ + if activation_fn is None or activation_fn in ['none', 'None']: + return None if activation_fn not in ACTIVATION_FN_DICT: raise RuntimeError(f"{activation_fn} is not a valid activation function") return ACTIVATION_FN_DICT[activation_fn] diff --git a/deepmd/descriptor/__init__.py b/deepmd/descriptor/__init__.py index dd022a4e08..e4df063a81 100644 --- a/deepmd/descriptor/__init__.py +++ b/deepmd/descriptor/__init__.py @@ -7,3 +7,4 @@ from .se_a_ef import DescrptSeAEf from .se_a_ef import DescrptSeAEfLower from .loc_frame import DescrptLocFrame +from .se_atten import DescrptSeAtten diff --git a/deepmd/descriptor/se_atten.py b/deepmd/descriptor/se_atten.py new file mode 100644 index 0000000000..ead4f93d46 --- /dev/null +++ b/deepmd/descriptor/se_atten.py @@ -0,0 +1,777 @@ +import math +import numpy as np +from typing import Tuple, List, Dict, Any +from packaging.version import Version + +from deepmd.env import tf +from deepmd.common import get_activation_func, get_precision, cast_precision +from deepmd.env import GLOBAL_TF_FLOAT_PRECISION +from deepmd.env import TF_VERSION +from deepmd.env import GLOBAL_NP_FLOAT_PRECISION +from deepmd.env import op_module +from deepmd.env import default_tf_session_config +from deepmd.utils.network import one_layer, embedding_net, embedding_net_rand_seed_shift +from deepmd.utils.tabulate import DPTabulate +from deepmd.utils.type_embed import embed_atom_type +from deepmd.utils.sess import run_sess +from deepmd.utils.graph import load_graph_def, get_tensor_by_name_from_graph, get_tensor_by_name +from deepmd.utils.errors import GraphWithoutTensorError +from .descriptor import Descriptor +from .se_a import DescrptSeA + + +@Descriptor.register("se_atten") +class DescrptSeAtten(DescrptSeA): + """ + Parameters + ---------- + rcut + The cut-off radius :math:`r_c` + rcut_smth + From where the environment matrix should be smoothed :math:`r_s` + sel : list[str] + sel[i] specifies the maxmum number of type i atoms in the cut-off radius + neuron : list[int] + Number of neurons in each hidden layers of the embedding net :math:`\mathcal{N}` + axis_neuron + Number of the axis neuron :math:`M_2` (number of columns of the sub-matrix of the embedding matrix) + resnet_dt + Time-step `dt` in the resnet construction: + y = x + dt * \phi (Wx + b) + trainable + If the weights of embedding net are trainable. + seed + Random seed for initializing the network parameters. + type_one_side + Try to build N_types embedding nets. Otherwise, building N_types^2 embedding nets + exclude_types : List[List[int]] + The excluded pairs of types which have no interaction with each other. + For example, `[[0, 1]]` means no interaction between type 0 and type 1. + set_davg_zero + Set the shift of embedding net input to zero. + activation_function + The activation function in the embedding net. Supported options are |ACTIVATION_FN| + precision + The precision of the embedding net parameters. Supported options are |PRECISION| + uniform_seed + Only for the purpose of backward compatibility, retrieves the old behavior of using the random seed + attn + The length of hidden vector during scale-dot attention computation. + attn_layer + The number of layers in attention mechanism. + attn_dotr + Whether to dot the relative coordinates on the attention weights as a gated scheme. + attn_mask + Whether to mask the diagonal in the attention weights. + """ + + def __init__(self, + rcut: float, + rcut_smth: float, + sel: int, + ntypes: int, + neuron: List[int] = [24, 48, 96], + axis_neuron: int = 8, + resnet_dt: bool = False, + trainable: bool = True, + seed: int = None, + type_one_side: bool = True, + exclude_types: List[List[int]] = [], + set_davg_zero: bool = False, + activation_function: str = 'tanh', + precision: str = 'default', + uniform_seed: bool = False, + attn: int = 128, + attn_layer: int = 2, + attn_dotr: bool = True, + attn_mask: bool = False + ) -> None: + DescrptSeA.__init__(self, + rcut, + rcut_smth, + [sel], + neuron=neuron, + axis_neuron=axis_neuron, + resnet_dt=resnet_dt, + trainable=trainable, + seed=seed, + type_one_side=type_one_side, + exclude_types=exclude_types, + set_davg_zero=set_davg_zero, + activation_function=activation_function, + precision=precision, + uniform_seed=uniform_seed + ) + """ + Constructor + """ + assert (Version(TF_VERSION) > Version('2')), "se_atten only support tensorflow version 2.0 or higher." + self.ntypes = ntypes + self.att_n = attn + self.attn_layer = attn_layer + self.attn_mask = attn_mask + self.attn_dotr = attn_dotr + + # descrpt config + self.sel_all_a = [sel] + self.sel_all_r = [0] + avg_zero = np.zeros([self.ntypes, self.ndescrpt]).astype(GLOBAL_NP_FLOAT_PRECISION) + std_ones = np.ones([self.ntypes, self.ndescrpt]).astype(GLOBAL_NP_FLOAT_PRECISION) + sub_graph = tf.Graph() + with sub_graph.as_default(): + name_pfx = 'd_sea_' + for ii in ['coord', 'box']: + self.place_holders[ii] = tf.placeholder(GLOBAL_NP_FLOAT_PRECISION, [None, None], + name=name_pfx + 't_' + ii) + self.place_holders['type'] = tf.placeholder(tf.int32, [None, None], name=name_pfx + 't_type') + self.place_holders['natoms_vec'] = tf.placeholder(tf.int32, [self.ntypes + 2], name=name_pfx + 't_natoms') + self.place_holders['default_mesh'] = tf.placeholder(tf.int32, [None], name=name_pfx + 't_mesh') + self.stat_descrpt, self.descrpt_deriv_t, self.rij_t, self.nlist_t, self.nei_type_vec_t, self.nmask_t \ + = op_module.prod_env_mat_a_mix(self.place_holders['coord'], + self.place_holders['type'], + self.place_holders['natoms_vec'], + self.place_holders['box'], + self.place_holders['default_mesh'], + tf.constant(avg_zero), + tf.constant(std_ones), + rcut_a=self.rcut_a, + rcut_r=self.rcut_r, + rcut_r_smth=self.rcut_r_smth, + sel_a=self.sel_all_a, + sel_r=self.sel_all_r) + self.sub_sess = tf.Session(graph=sub_graph, config=default_tf_session_config) + + def compute_input_stats(self, + data_coord: list, + data_box: list, + data_atype: list, + natoms_vec: list, + mesh: list, + input_dict: dict, + mixed_type: bool = False, + real_natoms_vec: list = None + ) -> None: + """ + Compute the statisitcs (avg and std) of the training data. The input will be normalized by the statistics. + + Parameters + ---------- + data_coord + The coordinates. Can be generated by deepmd.model.make_stat_input + data_box + The box. Can be generated by deepmd.model.make_stat_input + data_atype + The atom types. Can be generated by deepmd.model.make_stat_input + natoms_vec + The vector for the number of atoms of the system and different types of atoms. + If mixed_type is True, this para is blank. See real_natoms_vec. + mesh + The mesh for neighbor searching. Can be generated by deepmd.model.make_stat_input + input_dict + Dictionary for additional input + mixed_type + Whether to perform the mixed_type mode. + If True, the input data has the mixed_type format (see doc/model/train_se_atten.md), + in which frames in a system may have different natoms_vec(s), with the same nloc. + real_natoms_vec + If mixed_type is True, it takes in the real natoms_vec for each frame. + """ + all_davg = [] + all_dstd = [] + if True: + sumr = [] + suma = [] + sumn = [] + sumr2 = [] + suma2 = [] + if mixed_type: + sys_num = 0 + for cc, bb, tt, nn, mm, r_n in zip(data_coord, data_box, data_atype, natoms_vec, mesh, real_natoms_vec): + sysr, sysr2, sysa, sysa2, sysn \ + = self._compute_dstats_sys_smth(cc, bb, tt, nn, mm, mixed_type, r_n) + sys_num += 1 + sumr.append(sysr) + suma.append(sysa) + sumn.append(sysn) + sumr2.append(sysr2) + suma2.append(sysa2) + else: + for cc, bb, tt, nn, mm in zip(data_coord, data_box, data_atype, natoms_vec, mesh): + sysr, sysr2, sysa, sysa2, sysn \ + = self._compute_dstats_sys_smth(cc, bb, tt, nn, mm) + sumr.append(sysr) + suma.append(sysa) + sumn.append(sysn) + sumr2.append(sysr2) + suma2.append(sysa2) + sumr = np.sum(sumr, axis=0) + suma = np.sum(suma, axis=0) + sumn = np.sum(sumn, axis=0) + sumr2 = np.sum(sumr2, axis=0) + suma2 = np.sum(suma2, axis=0) + for type_i in range(self.ntypes): + davgunit = [sumr[type_i] / (sumn[type_i] + 1e-15), 0, 0, 0] + dstdunit = [self._compute_std(sumr2[type_i], sumr[type_i], sumn[type_i]), + self._compute_std(suma2[type_i], suma[type_i], sumn[type_i]), + self._compute_std(suma2[type_i], suma[type_i], sumn[type_i]), + self._compute_std(suma2[type_i], suma[type_i], sumn[type_i]) + ] + davg = np.tile(davgunit, self.ndescrpt // 4) + dstd = np.tile(dstdunit, self.ndescrpt // 4) + all_davg.append(davg) + all_dstd.append(dstd) + + if not self.set_davg_zero: + self.davg = np.array(all_davg) + self.dstd = np.array(all_dstd) + + def build(self, + coord_: tf.Tensor, + atype_: tf.Tensor, + natoms: tf.Tensor, + box_: tf.Tensor, + mesh: tf.Tensor, + input_dict: dict, + reuse: bool = None, + suffix: str = '' + ) -> tf.Tensor: + """ + Build the computational graph for the descriptor + + Parameters + ---------- + coord_ + The coordinate of atoms + atype_ + The type of atoms + natoms + The number of atoms. This tensor has the length of Ntypes + 2 + natoms[0]: number of local atoms + natoms[1]: total number of atoms held by this processor + natoms[i]: 2 <= i < Ntypes+2, number of type i atoms + mesh + For historical reasons, only the length of the Tensor matters. + if size of mesh == 6, pbc is assumed. + if size of mesh == 0, no-pbc is assumed. + input_dict + Dictionary for additional inputs + reuse + The weights in the networks should be reused when get the variable. + suffix + Name suffix to identify this descriptor + + Returns + ------- + descriptor + The output descriptor + """ + davg = self.davg + dstd = self.dstd + with tf.variable_scope('descrpt_attr' + suffix, reuse=reuse): + if davg is None: + davg = np.zeros([self.ntypes, self.ndescrpt]) + if dstd is None: + dstd = np.ones([self.ntypes, self.ndescrpt]) + t_rcut = tf.constant(np.max([self.rcut_r, self.rcut_a]), + name='rcut', + dtype=GLOBAL_TF_FLOAT_PRECISION) + t_ntypes = tf.constant(self.ntypes, + name='ntypes', + dtype=tf.int32) + t_ndescrpt = tf.constant(self.ndescrpt, + name='ndescrpt', + dtype=tf.int32) + t_sel = tf.constant(self.sel_a, + name='sel', + dtype=tf.int32) + t_original_sel = tf.constant(self.original_sel if self.original_sel is not None else self.sel_a, + name='original_sel', + dtype=tf.int32) + self.t_avg = tf.get_variable('t_avg', + davg.shape, + dtype=GLOBAL_TF_FLOAT_PRECISION, + trainable=False, + initializer=tf.constant_initializer(davg)) + self.t_std = tf.get_variable('t_std', + dstd.shape, + dtype=GLOBAL_TF_FLOAT_PRECISION, + trainable=False, + initializer=tf.constant_initializer(dstd)) + + with tf.control_dependencies([t_sel, t_original_sel]): + coord = tf.reshape(coord_, [-1, natoms[1] * 3]) + box = tf.reshape(box_, [-1, 9]) + atype = tf.reshape(atype_, [-1, natoms[1]]) + self.attn_weight = [None for i in range(self.attn_layer)] + self.angular_weight = [None for i in range(self.attn_layer)] + self.attn_weight_final = [None for i in range(self.attn_layer)] + self.G = None + self.qs = [None for i in range(self.attn_layer)] + self.ks = [None for i in range(self.attn_layer)] + self.vs = [None for i in range(self.attn_layer)] + + self.descrpt, self.descrpt_deriv, self.rij, self.nlist, self.nei_type_vec, self.nmask \ + = op_module.prod_env_mat_a_mix(coord, + atype, + natoms, + box, + mesh, + self.t_avg, + self.t_std, + rcut_a=self.rcut_a, + rcut_r=self.rcut_r, + rcut_r_smth=self.rcut_r_smth, + sel_a=self.sel_all_a, + sel_r=self.sel_all_r) + self.nei_type_vec = tf.reshape(self.nei_type_vec, [-1]) + self.nmask = tf.cast(tf.reshape(self.nmask, [-1, 1, self.sel_all_a[0]]), GLOBAL_TF_FLOAT_PRECISION) + self.negative_mask = -(2 << 32) * (1.0 - self.nmask) + # only used when tensorboard was set as true + tf.summary.histogram('descrpt', self.descrpt) + tf.summary.histogram('rij', self.rij) + tf.summary.histogram('nlist', self.nlist) + + self.descrpt_reshape = tf.reshape(self.descrpt, [-1, self.ndescrpt]) + self.atype_nloc = tf.reshape(tf.slice(atype, [0, 0], [-1, natoms[0]]), + [-1]) ## lammps will have error without this + self._identity_tensors(suffix=suffix) + + self.dout, self.qmat = self._pass_filter(self.descrpt_reshape, + self.atype_nloc, + natoms, + input_dict, + suffix=suffix, + reuse=reuse, + trainable=self.trainable) + + # only used when tensorboard was set as true + tf.summary.histogram('embedding_net_output', self.dout) + return self.dout + + def _pass_filter(self, + inputs, + atype, + natoms, + input_dict, + reuse=None, + suffix='', + trainable=True): + assert (input_dict is not None and input_dict.get('type_embedding', None) is not None), \ + 'se_atten desctiptor must use type_embedding' + type_embedding = input_dict.get('type_embedding', None) + inputs = tf.reshape(inputs, [-1, natoms[0], self.ndescrpt]) + output = [] + output_qmat = [] + inputs_i = inputs + inputs_i = tf.reshape(inputs_i, [-1, self.ndescrpt]) + type_i = -1 + layer, qmat = self._filter(inputs_i, type_i, natoms, name='filter_type_all' + suffix, reuse=reuse, + trainable=trainable, activation_fn=self.filter_activation_fn, + type_embedding=type_embedding, atype=atype) + layer = tf.reshape(layer, [tf.shape(inputs)[0], natoms[0], self.get_dim_out()]) + qmat = tf.reshape(qmat, [tf.shape(inputs)[0], natoms[0], self.get_dim_rot_mat_1() * 3]) + output.append(layer) + output_qmat.append(qmat) + output = tf.concat(output, axis=1) + output_qmat = tf.concat(output_qmat, axis=1) + return output, output_qmat + + def _compute_dstats_sys_smth(self, + data_coord, + data_box, + data_atype, + natoms_vec, + mesh, + mixed_type=False, + real_natoms_vec=None): + dd_all, descrpt_deriv_t, rij_t, nlist_t, nei_type_vec_t, nmask_t \ + = run_sess(self.sub_sess, [self.stat_descrpt, self.descrpt_deriv_t, self.rij_t, self.nlist_t, self.nei_type_vec_t, self.nmask_t], + feed_dict={ + self.place_holders['coord']: data_coord, + self.place_holders['type']: data_atype, + self.place_holders['natoms_vec']: natoms_vec, + self.place_holders['box']: data_box, + self.place_holders['default_mesh']: mesh, + }) + if mixed_type: + nframes = dd_all.shape[0] + sysr = [0. for i in range(self.ntypes)] + sysa = [0. for i in range(self.ntypes)] + sysn = [0 for i in range(self.ntypes)] + sysr2 = [0. for i in range(self.ntypes)] + sysa2 = [0. for i in range(self.ntypes)] + for ff in range(nframes): + natoms = real_natoms_vec[ff] + dd_ff = np.reshape(dd_all[ff], [-1, self.ndescrpt * natoms[0]]) + start_index = 0 + for type_i in range(self.ntypes): + end_index = start_index + self.ndescrpt * natoms[2 + type_i] # center atom split + dd = dd_ff[:, start_index:end_index] + dd = np.reshape(dd, [-1, self.ndescrpt]) # nframes * typen_atoms , nnei * 4 + start_index = end_index + # compute + dd = np.reshape(dd, [-1, 4]) # nframes * typen_atoms * nnei, 4 + ddr = dd[:, :1] + dda = dd[:, 1:] + sumr = np.sum(ddr) + suma = np.sum(dda) / 3. + sumn = dd.shape[0] + sumr2 = np.sum(np.multiply(ddr, ddr)) + suma2 = np.sum(np.multiply(dda, dda)) / 3. + sysr[type_i] += sumr + sysa[type_i] += suma + sysn[type_i] += sumn + sysr2[type_i] += sumr2 + sysa2[type_i] += suma2 + else: + natoms = natoms_vec + dd_all = np.reshape(dd_all, [-1, self.ndescrpt * natoms[0]]) + start_index = 0 + sysr = [] + sysa = [] + sysn = [] + sysr2 = [] + sysa2 = [] + for type_i in range(self.ntypes): + end_index = start_index + self.ndescrpt * natoms[2 + type_i] # center atom split + dd = dd_all[:, start_index:end_index] + dd = np.reshape(dd, [-1, self.ndescrpt]) # nframes * typen_atoms , nnei * 4 + start_index = end_index + # compute + dd = np.reshape(dd, [-1, 4]) # nframes * typen_atoms * nnei, 4 + ddr = dd[:, :1] + dda = dd[:, 1:] + sumr = np.sum(ddr) + suma = np.sum(dda) / 3. + sumn = dd.shape[0] + sumr2 = np.sum(np.multiply(ddr, ddr)) + suma2 = np.sum(np.multiply(dda, dda)) / 3. + sysr.append(sumr) + sysa.append(suma) + sysn.append(sumn) + sysr2.append(sumr2) + sysa2.append(suma2) + return sysr, sysr2, sysa, sysa2, sysn + + def _lookup_type_embedding( + self, + xyz_scatter, + natype, + type_embedding, + ): + '''Concatenate `type_embedding` of neighbors and `xyz_scatter`. + If not self.type_one_side, concatenate `type_embedding` of center atoms as well. + + Parameters + ---------- + xyz_scatter: + shape is [nframes*natoms[0]*self.nnei, 1] + nframes: + shape is [] + natoms: + shape is [1+1+self.ntypes] + type_embedding: + shape is [self.ntypes, Y] where Y=jdata['type_embedding']['neuron'][-1] + + Returns + ------- + embedding: + environment of each atom represented by embedding. + ''' + te_out_dim = type_embedding.get_shape().as_list()[-1] + self.test_type_embedding = type_embedding + self.test_nei_embed = tf.nn.embedding_lookup(type_embedding, + self.nei_type_vec) # shape is [self.nnei, 1+te_out_dim] + # nei_embed = tf.tile(nei_embed, (nframes * natoms[0], 1)) # shape is [nframes*natoms[0]*self.nnei, te_out_dim] + nei_embed = tf.reshape(self.test_nei_embed, [-1, te_out_dim]) + self.embedding_input = tf.concat([xyz_scatter, nei_embed], + 1) # shape is [nframes*natoms[0]*self.nnei, 1+te_out_dim] + if not self.type_one_side: + self.atm_embed = tf.nn.embedding_lookup(type_embedding, natype) # shape is [nframes*natoms[0], te_out_dim] + self.atm_embed = tf.tile(self.atm_embed, + [1, self.nnei]) # shape is [nframes*natoms[0], self.nnei*te_out_dim] + self.atm_embed = tf.reshape(self.atm_embed, + [-1, te_out_dim]) # shape is [nframes*natoms[0]*self.nnei, te_out_dim] + self.embedding_input_2 = tf.concat([self.embedding_input, self.atm_embed], + 1) # shape is [nframes*natoms[0]*self.nnei, 1+te_out_dim+te_out_dim] + return self.embedding_input_2 + return self.embedding_input + + def _feedforward(self, input_xyz, d_in, d_mid): + residual = input_xyz + input_xyz = tf.nn.relu(one_layer( + input_xyz, + d_mid, + name='c_ffn1', + reuse=tf.AUTO_REUSE, + seed=self.seed, + activation_fn=None, + precision=self.filter_precision, + trainable=True, + uniform_seed=self.uniform_seed)) + input_xyz = one_layer( + input_xyz, + d_in, + name='c_ffn2', + reuse=tf.AUTO_REUSE, + seed=self.seed, + activation_fn=None, + precision=self.filter_precision, + trainable=True, + uniform_seed=self.uniform_seed) + input_xyz += residual + input_xyz = tf.keras.layers.LayerNormalization()(input_xyz) + return input_xyz + + def _scaled_dot_attn(self, Q, K, V, temperature, input_r, dotr=False, do_mask=False, layer=0, save_weights=True): + attn = tf.matmul(Q / temperature, K, transpose_b=True) + attn *= self.nmask + attn += self.negative_mask + attn = tf.nn.softmax(attn, axis=-1) + attn *= tf.reshape(self.nmask, [-1, attn.shape[-1], 1]) + if save_weights: + self.attn_weight[layer] = attn[0] # atom 0 + if dotr: + angular_weight = tf.matmul(input_r, input_r, transpose_b=True) # normalized + attn *= angular_weight + if save_weights: + self.angular_weight[layer] = angular_weight[0] # atom 0 + self.attn_weight_final[layer] = attn[0] # atom 0 + if do_mask: + nei = int(attn.shape[-1]) + mask = tf.cast(tf.ones((nei, nei)) - tf.eye(nei), self.filter_precision) + attn *= mask + output = tf.matmul(attn, V) + return output + + def _attention_layers( + self, + input_xyz, + layer_num, + shape_i, + outputs_size, + input_r, + dotr=False, + do_mask=False, + trainable=True + ): + sd_k = tf.sqrt(tf.cast(1., dtype=self.filter_precision)) + self.G = tf.reshape(input_xyz, (-1, shape_i[1] // 4, outputs_size[-1]))[0] + for i in range(layer_num): + with tf.variable_scope('attention_layer{}_'.format(i), reuse=tf.AUTO_REUSE): + # input_xyz_in = tf.nn.l2_normalize(input_xyz, -1) + Q_c = one_layer( + input_xyz, + self.att_n, + name='c_query', + reuse=tf.AUTO_REUSE, + seed=self.seed, + activation_fn=None, + precision=self.filter_precision, + trainable=trainable, + uniform_seed=self.uniform_seed) + K_c = one_layer( + input_xyz, + self.att_n, + name='c_key', + reuse=tf.AUTO_REUSE, + seed=self.seed, + activation_fn=None, + precision=self.filter_precision, + trainable=trainable, + uniform_seed=self.uniform_seed) + V_c = one_layer( + input_xyz, + self.att_n, + name='c_value', + reuse=tf.AUTO_REUSE, + seed=self.seed, + activation_fn=None, + precision=self.filter_precision, + trainable=trainable, + uniform_seed=self.uniform_seed) + # # natom x nei_type_i x out_size + # xyz_scatter = tf.reshape(xyz_scatter, (-1, shape_i[1] // 4, outputs_size[-1])) + # natom x nei_type_i x att_n + Q_c = tf.nn.l2_normalize(tf.reshape(Q_c, (-1, shape_i[1] // 4, self.att_n)), -1) + K_c = tf.nn.l2_normalize(tf.reshape(K_c, (-1, shape_i[1] // 4, self.att_n)), -1) + V_c = tf.nn.l2_normalize(tf.reshape(V_c, (-1, shape_i[1] // 4, self.att_n)), -1) + # Q_c = tf.reshape(Q_c, (-1, shape_i[1] // 4, self.att_n)) + # K_c = tf.reshape(K_c, (-1, shape_i[1] // 4, self.att_n)) + # V_c = tf.reshape(V_c, (-1, shape_i[1] // 4, self.att_n)) + self.qs[i] = Q_c[0] + self.ks[i] = K_c[0] + self.vs[i] = V_c[0] + + input_att = self._scaled_dot_attn(Q_c, K_c, V_c, sd_k, input_r, dotr=dotr, do_mask=do_mask, layer=i) + input_att = tf.reshape(input_att, (-1, self.att_n)) + + # A_c = tf.nn.softmax(tf.matmul(Q_c, K_c, transpose_b=True)/sd_k) + # # (natom x nei_type_i) x att_n + # input_att = tf.reshape(tf.matmul(A_c, V_c), (-1, self.att_n)) + + # (natom x nei_type_i) x out_size + input_xyz += one_layer( + input_att, + outputs_size[-1], + name='c_out', + reuse=tf.AUTO_REUSE, + seed=self.seed, + activation_fn=None, + precision=self.filter_precision, + trainable=trainable, + uniform_seed=self.uniform_seed) + input_xyz = tf.keras.layers.LayerNormalization()(input_xyz) + # input_xyz = self._feedforward(input_xyz, outputs_size[-1], self.att_n) + return input_xyz + + def _filter_lower( + self, + type_i, + type_input, + start_index, + incrs_index, + inputs, + type_embedding=None, + atype=None, + is_exclude=False, + activation_fn=None, + bavg=0.0, + stddev=1.0, + trainable=True, + suffix='', + name='filter_', + reuse=None + ): + """ + input env matrix, returns R.G + """ + outputs_size = [1] + self.filter_neuron + # cut-out inputs + # with natom x (nei_type_i x 4) + inputs_i = tf.slice(inputs, + [0, start_index * 4], + [-1, incrs_index * 4]) + shape_i = inputs_i.get_shape().as_list() + natom = tf.shape(inputs_i)[0] + # with (natom x nei_type_i) x 4 + inputs_reshape = tf.reshape(inputs_i, [-1, 4]) + # with (natom x nei_type_i) x 1 + xyz_scatter = tf.reshape(tf.slice(inputs_reshape, [0, 0], [-1, 1]), [-1, 1]) + assert atype is not None, 'atype must exist!!' + type_embedding = tf.cast(type_embedding, self.filter_precision) + xyz_scatter = self._lookup_type_embedding( + xyz_scatter, atype, type_embedding) + if self.compress: + raise RuntimeError('compression of attention descriptor is not supported at the moment') + # natom x 4 x outputs_size + if (not is_exclude): + with tf.variable_scope(name, reuse=reuse): + # with (natom x nei_type_i) x out_size + xyz_scatter = embedding_net( + xyz_scatter, + self.filter_neuron, + self.filter_precision, + activation_fn=activation_fn, + resnet_dt=self.filter_resnet_dt, + name_suffix=suffix, + stddev=stddev, + bavg=bavg, + seed=self.seed, + trainable=trainable, + uniform_seed=self.uniform_seed, + initial_variables=self.embedding_net_variables, + mixed_prec=self.mixed_prec) + if (not self.uniform_seed) and (self.seed is not None): self.seed += self.seed_shift + input_r = tf.slice(tf.reshape(inputs_i, (-1, shape_i[1] // 4, 4)), [0, 0, 1], [-1, -1, 3]) + input_r = tf.nn.l2_normalize(input_r, -1) + # natom x nei_type_i x out_size + xyz_scatter_att = tf.reshape( + self._attention_layers(xyz_scatter, self.attn_layer, shape_i, outputs_size, input_r, + dotr=self.attn_dotr, do_mask=self.attn_mask, trainable=trainable), + (-1, shape_i[1] // 4, outputs_size[-1])) + # xyz_scatter = tf.reshape(xyz_scatter, (-1, shape_i[1] // 4, outputs_size[-1])) + else: + # we can safely return the final xyz_scatter filled with zero directly + return tf.cast(tf.fill((natom, 4, outputs_size[-1]), 0.), self.filter_precision) + # When using tf.reshape(inputs_i, [-1, shape_i[1]//4, 4]) below + # [588 24] -> [588 6 4] correct + # but if sel is zero + # [588 0] -> [147 0 4] incorrect; the correct one is [588 0 4] + # So we need to explicitly assign the shape to tf.shape(inputs_i)[0] instead of -1 + return tf.matmul(tf.reshape(inputs_i, [natom, shape_i[1] // 4, 4]), xyz_scatter_att, transpose_a=True) + + @cast_precision + def _filter( + self, + inputs, + type_input, + natoms, + type_embedding=None, + atype=None, + activation_fn=tf.nn.tanh, + stddev=1.0, + bavg=0.0, + name='linear', + reuse=None, + trainable=True): + nframes = tf.shape(tf.reshape(inputs, [-1, natoms[0], self.ndescrpt]))[0] + # natom x (nei x 4) + shape = inputs.get_shape().as_list() + outputs_size = [1] + self.filter_neuron + outputs_size_2 = self.n_axis_neuron + all_excluded = all([(type_input, type_i) in self.exclude_types for type_i in range(self.ntypes)]) + if all_excluded: + # all types are excluded so result and qmat should be zeros + # we can safaly return a zero matrix... + # See also https://stackoverflow.com/a/34725458/9567349 + # result: natom x outputs_size x outputs_size_2 + # qmat: natom x outputs_size x 3 + natom = tf.shape(inputs)[0] + result = tf.cast(tf.fill((natom, outputs_size_2, outputs_size[-1]), 0.), GLOBAL_TF_FLOAT_PRECISION) + qmat = tf.cast(tf.fill((natom, outputs_size[-1], 3), 0.), GLOBAL_TF_FLOAT_PRECISION) + return result, qmat + + start_index = 0 + type_i = 0 + # natom x 4 x outputs_size + xyz_scatter_1 = self._filter_lower( + type_i, type_input, + start_index, np.cumsum(self.sel_a)[-1], + inputs, + type_embedding=type_embedding, + is_exclude=False, + activation_fn=activation_fn, + stddev=stddev, + bavg=bavg, + trainable=trainable, + name=name, + reuse=reuse, + atype=atype) + # natom x nei x outputs_size + # xyz_scatter = tf.concat(xyz_scatter_total, axis=1) + # natom x nei x 4 + # inputs_reshape = tf.reshape(inputs, [-1, shape[1]//4, 4]) + # natom x 4 x outputs_size + # xyz_scatter_1 = tf.matmul(inputs_reshape, xyz_scatter, transpose_a = True) + if self.original_sel is None: + # shape[1] = nnei * 4 + nnei = shape[1] / 4 + else: + nnei = tf.cast(tf.Variable(np.sum(self.original_sel), dtype=tf.int32, trainable=False, name="nnei"), + self.filter_precision) + xyz_scatter_1 = xyz_scatter_1 / nnei + # natom x 4 x outputs_size_2 + xyz_scatter_2 = tf.slice(xyz_scatter_1, [0, 0, 0], [-1, -1, outputs_size_2]) + # # natom x 3 x outputs_size_2 + # qmat = tf.slice(xyz_scatter_2, [0,1,0], [-1, 3, -1]) + # natom x 3 x outputs_size_1 + qmat = tf.slice(xyz_scatter_1, [0, 1, 0], [-1, 3, -1]) + # natom x outputs_size_1 x 3 + qmat = tf.transpose(qmat, perm=[0, 2, 1]) + # natom x outputs_size x outputs_size_2 + result = tf.matmul(xyz_scatter_1, xyz_scatter_2, transpose_a=True) + # natom x (outputs_size x outputs_size_2) + result = tf.reshape(result, [-1, outputs_size_2 * outputs_size[-1]]) + + return result, qmat diff --git a/deepmd/entrypoints/train.py b/deepmd/entrypoints/train.py index edc76b4d85..7cdb0307d4 100755 --- a/deepmd/entrypoints/train.py +++ b/deepmd/entrypoints/train.py @@ -90,7 +90,10 @@ def train( jdata = normalize(jdata) - if not is_compress and not skip_neighbor_stat: + if jdata['model']['descriptor']['type'] in ['se_atten'] and isinstance(jdata['model']['descriptor']['sel'], list): + jdata['model']['descriptor']['sel'] = sum(jdata['model']['descriptor']['sel']) + + if not is_compress and not skip_neighbor_stat and jdata['model']['descriptor']['type'] not in ['se_atten']: jdata = update_sel(jdata) with open(output, "w") as fp: diff --git a/deepmd/fit/ener.py b/deepmd/fit/ener.py index 61d70045d8..876af78493 100644 --- a/deepmd/fit/ener.py +++ b/deepmd/fit/ener.py @@ -168,7 +168,8 @@ def get_numb_aparam(self) -> int: return self.numb_fparam def compute_output_stats(self, - all_stat: dict + all_stat: dict, + mixed_type: bool = False ) -> None: """ Compute the ouput statistics @@ -179,10 +180,14 @@ def compute_output_stats(self, must have the following components: all_stat['energy'] of shape n_sys x n_batch x n_frame can be prepared by model.make_stat_input + mixed_type + Whether to perform the mixed_type mode. + If True, the input data has the mixed_type format (see doc/model/train_se_atten.md), + in which frames in a system may have different natoms_vec(s), with the same nloc. """ - self.bias_atom_e = self._compute_output_stats(all_stat, rcond = self.rcond) + self.bias_atom_e = self._compute_output_stats(all_stat, rcond=self.rcond, mixed_type=mixed_type) - def _compute_output_stats(self, all_stat, rcond = 1e-3): + def _compute_output_stats(self, all_stat, rcond=1e-3, mixed_type=False): data = all_stat['energy'] # data[sys_idx][batch_idx][frame_idx] sys_ener = np.array([]) @@ -193,11 +198,22 @@ def _compute_output_stats(self, all_stat, rcond = 1e-3): sys_data.append(data[ss][ii][jj]) sys_data = np.concatenate(sys_data) sys_ener = np.append(sys_ener, np.average(sys_data)) - data = all_stat['natoms_vec'] sys_tynatom = np.array([]) - nsys = len(data) - for ss in range(len(data)): - sys_tynatom = np.append(sys_tynatom, data[ss][0].astype(np.float64)) + if mixed_type: + data = all_stat['real_natoms_vec'] + nsys = len(data) + for ss in range(len(data)): + tmp_tynatom = [] + for ii in range(len(data[ss])): + for jj in range(len(data[ss][ii])): + tmp_tynatom.append(data[ss][ii][jj].astype(np.float64)) + tmp_tynatom = np.average(np.array(tmp_tynatom), axis=0) + sys_tynatom = np.append(sys_tynatom, tmp_tynatom) + else: + data = all_stat['natoms_vec'] + nsys = len(data) + for ss in range(len(data)): + sys_tynatom = np.append(sys_tynatom, data[ss][0].astype(np.float64)) sys_tynatom = np.reshape(sys_tynatom, [nsys,-1]) sys_tynatom = sys_tynatom[:,2:] if len(self.atom_ener) > 0: @@ -402,6 +418,11 @@ def build (self, t_daparam = tf.constant(self.numb_aparam, name = 'daparam', dtype = tf.int32) + self.t_bias_atom_e = tf.get_variable('t_bias_atom_e', + self.bias_atom_e.shape, + dtype=GLOBAL_TF_FLOAT_PRECISION, + trainable=False, + initializer=tf.constant_initializer(self.bias_atom_e)) if self.numb_fparam > 0: t_fparam_avg = tf.get_variable('t_fparam_avg', self.numb_fparam, @@ -452,12 +473,16 @@ def build (self, aparam = tf.reshape(aparam, [-1, self.numb_aparam * natoms[0]]) type_embedding = input_dict.get('type_embedding', None) + atype = input_dict.get('atype', None) if type_embedding is not None: - atype_embed = embed_atom_type(self.ntypes, natoms, type_embedding) - atype_embed = tf.tile(atype_embed,[tf.shape(inputs)[0],1]) + atype_nall = tf.reshape(atype, [-1, natoms[1]]) + self.atype_nloc = tf.reshape(tf.slice(atype_nall, [0, 0], [-1, natoms[0]]), [-1]) ## lammps will make error + atype_embed = tf.nn.embedding_lookup(type_embedding, self.atype_nloc) else: atype_embed = None + self.atype_embed = atype_embed + if atype_embed is None: start_index = 0 outs_list = [] @@ -503,11 +528,11 @@ def build (self, bias_atom_e=0.0, suffix=suffix, reuse=reuse ) outs = tf.reshape(final_layer, [tf.shape(inputs)[0], natoms[0]]) - # add atom energy bias; TF will broadcast to all batches - # tf.repeat is avaiable in TF>=2.1 or TF 1.15 - _TF_VERSION = Version(TF_VERSION) - if (Version('1.15') <= _TF_VERSION < Version('2') or _TF_VERSION >= Version('2.1')) and self.bias_atom_e is not None: - outs += tf.repeat(tf.Variable(self.bias_atom_e, dtype=self.fitting_precision, trainable=False, name="bias_atom_ei"), natoms[2:]) + # add bias + self.atom_ener_before = outs + self.add_type = tf.reshape(tf.nn.embedding_lookup(self.t_bias_atom_e, self.atype_nloc), [tf.shape(inputs)[0], natoms[0]]) + outs = outs + self.add_type + self.atom_ener_after = outs if self.tot_ener_zero: force_tot_ener = 0.0 diff --git a/deepmd/model/ener.py b/deepmd/model/ener.py index 471a0b4743..7504e22906 100644 --- a/deepmd/model/ener.py +++ b/deepmd/model/ener.py @@ -92,23 +92,36 @@ def get_type_map (self) : def data_stat(self, data): all_stat = make_stat_input(data, self.data_stat_nbatch, merge_sys = False) m_all_stat = merge_sys_stat(all_stat) - self._compute_input_stat(m_all_stat, protection = self.data_stat_protect) - self._compute_output_stat(all_stat) + self._compute_input_stat(m_all_stat, protection=self.data_stat_protect, mixed_type=data.mixed_type) + self._compute_output_stat(all_stat, mixed_type=data.mixed_type) # self.bias_atom_e = data.compute_energy_shift(self.rcond) - def _compute_input_stat (self, all_stat, protection = 1e-2) : - self.descrpt.compute_input_stats(all_stat['coord'], - all_stat['box'], - all_stat['type'], - all_stat['natoms_vec'], - all_stat['default_mesh'], - all_stat) - self.fitting.compute_input_stats(all_stat, protection = protection) + def _compute_input_stat (self, all_stat, protection=1e-2, mixed_type=False): + if mixed_type: + self.descrpt.compute_input_stats(all_stat['coord'], + all_stat['box'], + all_stat['type'], + all_stat['natoms_vec'], + all_stat['default_mesh'], + all_stat, + mixed_type, + all_stat['real_natoms_vec']) + else: + self.descrpt.compute_input_stats(all_stat['coord'], + all_stat['box'], + all_stat['type'], + all_stat['natoms_vec'], + all_stat['default_mesh'], + all_stat) + self.fitting.compute_input_stats(all_stat, protection=protection) + + def _compute_output_stat (self, all_stat, mixed_type=False): + if mixed_type: + self.fitting.compute_output_stats(all_stat, mixed_type=mixed_type) + else: + self.fitting.compute_output_stats(all_stat) - def _compute_output_stat (self, all_stat) : - self.fitting.compute_output_stats(all_stat) - def build (self, coord_, atype_, @@ -158,6 +171,7 @@ def build (self, suffix = suffix, ) input_dict['type_embedding'] = type_embedding + input_dict['atype'] = atype_ if frz_model == None: dout \ @@ -195,6 +209,7 @@ def build (self, input_dict, reuse = reuse, suffix = suffix) + self.atom_ener = atom_ener if self.srtab is not None : sw_lambda, sw_deriv \ diff --git a/deepmd/train/trainer.py b/deepmd/train/trainer.py index 86edb8db78..52cee2427b 100644 --- a/deepmd/train/trainer.py +++ b/deepmd/train/trainer.py @@ -24,6 +24,7 @@ from deepmd.utils.sess import run_sess from deepmd.utils.type_embed import TypeEmbedNet from deepmd.utils.graph import load_graph_def, get_tensor_by_name_from_graph +from deepmd.utils.argcheck import type_embedding_args from tensorflow.python.client import timeline from deepmd.env import op_module, TF_VERSION @@ -69,17 +70,20 @@ def _init_param(self, jdata): # nvnmd self.nvnmd_param = jdata.get('nvnmd', {}) nvnmd_cfg.init_from_jdata(self.nvnmd_param) - nvnmd_cfg.init_from_deepmd_input(model_param) if nvnmd_cfg.enable: + nvnmd_cfg.init_from_deepmd_input(model_param) nvnmd_cfg.disp_message() nvnmd_cfg.save() # descriptor try: descrpt_type = descrpt_param['type'] + self.descrpt_type = descrpt_type except KeyError: raise KeyError('the type of descriptor should be set by `type`') + if descrpt_param['type'] in ['se_atten']: + descrpt_param['ntypes'] = len(model_param['type_map']) self.descrpt = Descriptor(**descrpt_param) # fitting net @@ -112,6 +116,9 @@ def _init_param(self, jdata): raise RuntimeError('unknow fitting type ' + fitting_type) # type embedding + padding = False + if descrpt_type == 'se_atten': + padding = True if typeebd_param is not None: self.typeebd = TypeEmbedNet( neuron=typeebd_param['neuron'], @@ -119,7 +126,20 @@ def _init_param(self, jdata): activation_function=typeebd_param['activation_function'], precision=typeebd_param['precision'], trainable=typeebd_param['trainable'], - seed=typeebd_param['seed'] + seed=typeebd_param['seed'], + padding=padding + ) + elif descrpt_type == 'se_atten': + default_args = type_embedding_args() + default_args_dict = {i.name: i.default for i in default_args} + self.typeebd = TypeEmbedNet( + neuron=default_args_dict['neuron'], + resnet_dt=default_args_dict['resnet_dt'], + activation_function=None, + precision=default_args_dict['precision'], + trainable=default_args_dict['trainable'], + seed=default_args_dict['seed'], + padding=padding ) else: self.typeebd = None @@ -272,6 +292,10 @@ def build (self, self.ntypes = self.model.get_ntypes() self.stop_batch = stop_batch + if not self.is_compress and data.mixed_type: + assert self.descrpt_type in ['se_atten'], 'Data in mixed_type format must use attention descriptor!' + assert self.fitting_type in ['ener'], 'Data in mixed_type format must use ener fitting!' + if self.numb_fparam > 0 : log.info("training with %d frame parameter(s)" % self.numb_fparam) else: @@ -585,7 +609,7 @@ def save_checkpoint(self, cur_batch: int): def get_feed_dict(self, batch, is_training): feed_dict = {} for kk in batch.keys(): - if kk == 'find_type' or kk == 'type': + if kk == 'find_type' or kk == 'type' or kk == 'real_natoms_vec': continue if 'find_' in kk: feed_dict[self.place_holders[kk]] = batch[kk] diff --git a/deepmd/utils/argcheck.py b/deepmd/utils/argcheck.py index e7c7edb170..7ba62f9953 100644 --- a/deepmd/utils/argcheck.py +++ b/deepmd/utils/argcheck.py @@ -28,17 +28,17 @@ def type_embedding_args(): doc_neuron = 'Number of neurons in each hidden layers of the embedding net. When two layers are of the same size or one layer is twice as large as the previous layer, a skip connection is built.' doc_resnet_dt = 'Whether to use a "Timestep" in the skip connection' doc_seed = 'Random seed for parameter initialization' - doc_activation_function = f'The activation function in the embedding net. Supported activation functions are {list_to_doc(ACTIVATION_FN_DICT.keys())}. Note that "gelu" denotes the custom operator version, and "gelu_tf" denotes the TF standard version.' + doc_activation_function = f'The activation function in the embedding net. Supported activation functions are {list_to_doc(ACTIVATION_FN_DICT.keys())} Note that "gelu" denotes the custom operator version, and "gelu_tf" denotes the TF standard version.' doc_precision = f'The precision of the embedding net parameters, supported options are {list_to_doc(PRECISION_DICT.keys())} Default follows the interface precision.' doc_trainable = 'If the parameters in the embedding net are trainable' return [ - Argument("neuron", list, optional = True, default = [2, 4, 8], doc = doc_neuron), + Argument("neuron", list, optional = True, default = [8], doc = doc_neuron), Argument("activation_function", str, optional = True, default = 'tanh', doc = doc_activation_function), Argument("resnet_dt", bool, optional = True, default = False, doc = doc_resnet_dt), Argument("precision", str, optional = True, default = "default", doc = doc_precision), Argument("trainable", bool, optional = True, default = True, doc = doc_trainable), - Argument("seed", [int,None], optional = True, doc = doc_seed), + Argument("seed", [int,None], optional = True, default = None, doc = doc_seed), ] @@ -128,7 +128,7 @@ def descrpt_se_a_args(): doc_rcut_smth = 'Where to start smoothing. For example the 1/r term is smoothed from `rcut` to `rcut_smth`' doc_neuron = 'Number of neurons in each hidden layers of the embedding net. When two layers are of the same size or one layer is twice as large as the previous layer, a skip connection is built.' doc_axis_neuron = 'Size of the submatrix of G (embedding matrix).' - doc_activation_function = f'The activation function in the embedding net. Supported activation functions are {list_to_doc(ACTIVATION_FN_DICT.keys())}. Note that "gelu" denotes the custom operator version, and "gelu_tf" denotes the TF standard version.' + doc_activation_function = f'The activation function in the embedding net. Supported activation functions are {list_to_doc(ACTIVATION_FN_DICT.keys())} Note that "gelu" denotes the custom operator version, and "gelu_tf" denotes the TF standard version.' doc_resnet_dt = 'Whether to use a "Timestep" in the skip connection' doc_type_one_side = 'Try to build N_types embedding nets. Otherwise, building N_types^2 embedding nets' doc_precision = f'The precision of the embedding net parameters, supported options are {list_to_doc(PRECISION_DICT.keys())} Default follows the interface precision.' @@ -162,7 +162,7 @@ def descrpt_se_t_args(): doc_rcut = 'The cut-off radius.' doc_rcut_smth = 'Where to start smoothing. For example the 1/r term is smoothed from `rcut` to `rcut_smth`' doc_neuron = 'Number of neurons in each hidden layers of the embedding net. When two layers are of the same size or one layer is twice as large as the previous layer, a skip connection is built.' - doc_activation_function = f'The activation function in the embedding net. Supported activation functions are {list_to_doc(ACTIVATION_FN_DICT.keys())}. Note that "gelu" denotes the custom operator version, and "gelu_tf" denotes the TF standard version.' + doc_activation_function = f'The activation function in the embedding net. Supported activation functions are {list_to_doc(ACTIVATION_FN_DICT.keys())} Note that "gelu" denotes the custom operator version, and "gelu_tf" denotes the TF standard version.' doc_resnet_dt = 'Whether to use a "Timestep" in the skip connection' doc_precision = f'The precision of the embedding net parameters, supported options are {list_to_doc(PRECISION_DICT.keys())} Default follows the interface precision.' doc_trainable = 'If the parameters in the embedding net are trainable' @@ -205,7 +205,7 @@ def descrpt_se_r_args(): doc_rcut = 'The cut-off radius.' doc_rcut_smth = 'Where to start smoothing. For example the 1/r term is smoothed from `rcut` to `rcut_smth`' doc_neuron = 'Number of neurons in each hidden layers of the embedding net. When two layers are of the same size or one layer is twice as large as the previous layer, a skip connection is built.' - doc_activation_function = f'The activation function in the embedding net. Supported activation functions are {list_to_doc(ACTIVATION_FN_DICT.keys())}. Note that "gelu" denotes the custom operator version, and "gelu_tf" denotes the TF standard version.' + doc_activation_function = f'The activation function in the embedding net. Supported activation functions are {list_to_doc(ACTIVATION_FN_DICT.keys())} Note that "gelu" denotes the custom operator version, and "gelu_tf" denotes the TF standard version.' doc_resnet_dt = 'Whether to use a "Timestep" in the skip connection' doc_type_one_side = 'Try to build N_types embedding nets. Otherwise, building N_types^2 embedding nets' doc_precision = f'The precision of the embedding net parameters, supported options are {list_to_doc(PRECISION_DICT.keys())} Default follows the interface precision.' @@ -239,6 +239,48 @@ def descrpt_hybrid_args(): ] +@descrpt_args_plugin.register("se_atten") +def descrpt_se_atten_args(): + doc_sel = 'This parameter set the number of selected neighbors. Note that this parameter is a little different from that in other descriptors. Instead of separating each type of atoms, only the summation matters. And this number is highly related with the efficiency, thus one should not make it too large. Usually 200 or less is enough, far away from the GPU limitation 4096. It can be:\n\n\ + - `int`. The maximum number of neighbor atoms to be considered. We recommend it to be less than 200. \n\n\ + - `List[int]`. The length of the list should be the same as the number of atom types in the system. `sel[i]` gives the selected number of type-i neighbors. Only the summation of `sel[i]` matters, and it is recommended to be less than 200.' + doc_rcut = 'The cut-off radius.' + doc_rcut_smth = 'Where to start smoothing. For example the 1/r term is smoothed from `rcut` to `rcut_smth`' + doc_neuron = 'Number of neurons in each hidden layers of the embedding net. When two layers are of the same size or one layer is twice as large as the previous layer, a skip connection is built.' + doc_axis_neuron = 'Size of the submatrix of G (embedding matrix).' + doc_activation_function = f'The activation function in the embedding net. Supported activation functions are {list_to_doc(ACTIVATION_FN_DICT.keys())} Note that "gelu" denotes the custom operator version, and "gelu_tf" denotes the TF standard version.' + doc_resnet_dt = 'Whether to use a "Timestep" in the skip connection' + doc_type_one_side = 'Whether to consider the information from only one side or both sides.' + doc_precision = f'The precision of the embedding net parameters, supported options are {list_to_doc(PRECISION_DICT.keys())} Default follows the interface precision.' + doc_trainable = 'If the parameters in the embedding net is trainable' + doc_seed = 'Random seed for parameter initialization' + doc_exclude_types = 'The excluded pairs of types which have no interaction with each other. For example, `[[0, 1]]` means no interaction between type 0 and type 1.' + doc_set_davg_zero = 'Set the normalization average to zero. This option should be set when `atom_ener` in the energy fitting is used' + doc_attn = 'The length of hidden vectors in attention layers' + doc_attn_layer = 'The number of attention layers' + doc_attn_dotr = 'Whether to do dot product with the normalized relative coordinates' + doc_attn_mask = 'Whether to do mask on the diagonal in the attention matrix' + + return [ + Argument("sel", [int, list], optional=True, default=200, doc=doc_sel), + Argument("rcut", float, optional=True, default=6.0, doc=doc_rcut), + Argument("rcut_smth", float, optional=True, default=0.5, doc=doc_rcut_smth), + Argument("neuron", list, optional=True, default=[10, 20, 40], doc=doc_neuron), + Argument("axis_neuron", int, optional=True, default=4, alias=['n_axis_neuron'], doc=doc_axis_neuron), + Argument("activation_function", str, optional=True, default='tanh', doc=doc_activation_function), + Argument("resnet_dt", bool, optional=True, default=False, doc=doc_resnet_dt), + Argument("type_one_side", bool, optional=True, default=False, doc=doc_type_one_side), + Argument("precision", str, optional=True, default="default", doc=doc_precision), + Argument("trainable", bool, optional=True, default=True, doc=doc_trainable), + Argument("seed", [int, None], optional=True, doc=doc_seed), + Argument("exclude_types", list, optional=True, default=[], doc=doc_exclude_types), + Argument("set_davg_zero", bool, optional=True, default=False, doc=doc_set_davg_zero), + Argument("attn", int, optional=True, default=100, doc=doc_attn), + Argument("attn_layer", int, optional=True, default=4, doc=doc_attn_layer), + Argument("attn_dotr", bool, optional=True, default=False, doc=doc_attn_dotr), + Argument("attn_mask", bool, optional=True, default=False, doc=doc_attn_mask) + ] + def descrpt_variant_type_args(exclude_hybrid: bool = False) -> Variant: link_lf = make_link('loc_frame', 'model/descriptor[loc_frame]') link_se_e2_a = make_link('se_e2_a', 'model/descriptor[se_e2_a]') @@ -246,12 +288,14 @@ def descrpt_variant_type_args(exclude_hybrid: bool = False) -> Variant: link_se_e3 = make_link('se_e3', 'model/descriptor[se_e3]') link_se_a_tpe = make_link('se_a_tpe', 'model/descriptor[se_a_tpe]') link_hybrid = make_link('hybrid', 'model/descriptor[hybrid]') + link_se_atten = make_link('se_atten', 'model/descriptor[se_atten]') doc_descrpt_type = f'The type of the descritpor. See explanation below. \n\n\ - `loc_frame`: Defines a local frame at each atom, and the compute the descriptor as local coordinates under this frame.\n\n\ - `se_e2_a`: Used by the smooth edition of Deep Potential. The full relative coordinates are used to construct the descriptor.\n\n\ - `se_e2_r`: Used by the smooth edition of Deep Potential. Only the distance between atoms is used to construct the descriptor.\n\n\ - `se_e3`: Used by the smooth edition of Deep Potential. The full relative coordinates are used to construct the descriptor. Three-body embedding will be used by this descriptor.\n\n\ - `se_a_tpe`: Used by the smooth edition of Deep Potential. The full relative coordinates are used to construct the descriptor. Type embedding will be used by this descriptor.\n\n\ +- `se_atten`: Used by the smooth edition of Deep Potential. The full relative coordinates are used to construct the descriptor. Attention mechanism will be used by this descriptor.\n\n\ - `hybrid`: Concatenate of a list of descriptors as a new descriptor.' return Variant("type", descrpt_args_plugin.get_all_argument(), doc = doc_descrpt_type) @@ -262,7 +306,7 @@ def fitting_ener(): doc_numb_fparam = 'The dimension of the frame parameter. If set to >0, file `fparam.npy` should be included to provided the input fparams.' doc_numb_aparam = 'The dimension of the atomic parameter. If set to >0, file `aparam.npy` should be included to provided the input aparams.' doc_neuron = 'The number of neurons in each hidden layers of the fitting net. When two hidden layers are of the same size, a skip connection is built.' - doc_activation_function = f'The activation function in the fitting net. Supported activation functions are {list_to_doc(ACTIVATION_FN_DICT.keys())}. Note that "gelu" denotes the custom operator version, and "gelu_tf" denotes the TF standard version.' + doc_activation_function = f'The activation function in the fitting net. Supported activation functions are {list_to_doc(ACTIVATION_FN_DICT.keys())} Note that "gelu" denotes the custom operator version, and "gelu_tf" denotes the TF standard version.' doc_precision = f'The precision of the fitting net parameters, supported options are {list_to_doc(PRECISION_DICT.keys())} Default follows the interface precision.' doc_resnet_dt = 'Whether to use a "Timestep" in the skip connection' doc_trainable = 'Whether the parameters in the fitting net are trainable. This option can be\n\n\ @@ -288,7 +332,7 @@ def fitting_ener(): def fitting_polar(): doc_neuron = 'The number of neurons in each hidden layers of the fitting net. When two hidden layers are of the same size, a skip connection is built.' - doc_activation_function = f'The activation function in the fitting net. Supported activation functions are {list_to_doc(ACTIVATION_FN_DICT.keys())}. Note that "gelu" denotes the custom operator version, and "gelu_tf" denotes the TF standard version.' + doc_activation_function = f'The activation function in the fitting net. Supported activation functions are {list_to_doc(ACTIVATION_FN_DICT.keys())} Note that "gelu" denotes the custom operator version, and "gelu_tf" denotes the TF standard version.' doc_resnet_dt = 'Whether to use a "Timestep" in the skip connection' doc_precision = f'The precision of the fitting net parameters, supported options are {list_to_doc(PRECISION_DICT.keys())} Default follows the interface precision.' doc_scale = 'The output of the fitting net (polarizability matrix) will be scaled by ``scale``' @@ -320,7 +364,7 @@ def fitting_polar(): def fitting_dipole(): doc_neuron = 'The number of neurons in each hidden layers of the fitting net. When two hidden layers are of the same size, a skip connection is built.' - doc_activation_function = f'The activation function in the fitting net. Supported activation functions are {list_to_doc(ACTIVATION_FN_DICT.keys())}. Note that "gelu" denotes the custom operator version, and "gelu_tf" denotes the TF standard version.' + doc_activation_function = f'The activation function in the fitting net. Supported activation functions are {list_to_doc(ACTIVATION_FN_DICT.keys())} Note that "gelu" denotes the custom operator version, and "gelu_tf" denotes the TF standard version.' doc_resnet_dt = 'Whether to use a "Timestep" in the skip connection' doc_precision = f'The precision of the fitting net parameters, supported options are {list_to_doc(PRECISION_DICT.keys())} Default follows the interface precision.' doc_sel_type = 'The atom types for which the atomic dipole will be provided. If not set, all types will be selected.' diff --git a/deepmd/utils/data.py b/deepmd/utils/data.py index 0709352d7a..b5faf15c34 100644 --- a/deepmd/utils/data.py +++ b/deepmd/utils/data.py @@ -48,6 +48,7 @@ def __init__ (self, root = DPPath(sys_path) self.dirs = root.glob(set_prefix + ".*") self.dirs.sort() + self.mixed_type = self._check_mode(self.dirs[0]) # mixed_type format only has one set # load atom type self.atom_type = self._load_type(root) self.natoms = len(self.atom_type) @@ -59,9 +60,12 @@ def __init__ (self, self.pbc = self._check_pbc(root) # enforce type_map if necessary if type_map is not None and self.type_map is not None: - atom_type_ = [type_map.index(self.type_map[ii]) for ii in self.atom_type] - self.atom_type = np.array(atom_type_, dtype = np.int32) + if not self.mixed_type: + atom_type_ = [type_map.index(self.type_map[ii]) for ii in self.atom_type] + self.atom_type = np.array(atom_type_, dtype = np.int32) self.type_map = type_map + if type_map is None and self.type_map is None and self.mixed_type: + raise RuntimeError('mixed_type format must have type_map!') # make idx map self.idx_map = self._make_idx_map(self.atom_type) # train dirs @@ -408,8 +412,7 @@ def _shuffle_data (self, if type(data[kk]) == np.ndarray and \ len(data[kk].shape) == 2 and \ data[kk].shape[0] == nframes and \ - not('find_' in kk) and \ - 'type' != kk: + not('find_' in kk): ret[kk] = data[kk][idx] else : ret[kk] = data[kk] @@ -430,7 +433,6 @@ def _load_set(self, set_name: DPPath) : assert(coord.shape[1] == self.data_dict['coord']['ndof'] * self.natoms) # load keys data = {} - data['type'] = np.tile (self.atom_type[self.idx_map], (nframes, 1)) for kk in self.data_dict.keys(): if self.data_dict[kk]['reduce'] is None : data['find_'+kk], data[kk] \ @@ -453,6 +455,17 @@ def _load_set(self, set_name: DPPath) : tmp_in = data[k_in].astype(GLOBAL_ENER_FLOAT_PRECISION) data[kk] = np.sum(np.reshape(tmp_in, [nframes, self.natoms, ndof]), axis = 1) + if self.mixed_type: + type_path = set_name / "real_atom_types.npy" + real_type = type_path.load_numpy().astype(np.int32).reshape([nframes, -1]) + data['type'] = real_type + natoms = data['type'].shape[1] + data['real_natoms_vec'] = np.concatenate((np.tile(np.array([natoms, natoms], dtype=np.int32), (nframes, 1)), + np.array([(real_type == i).sum(axis=-1) for i in range(self.get_ntypes())], + dtype=np.int32).T), axis=-1) + else: + data['type'] = np.tile(self.atom_type[self.idx_map], (nframes, 1)) + return data @@ -524,6 +537,9 @@ def _check_pbc(self, sys_path: DPPath): pbc = False return pbc + def _check_mode(self, set_path: DPPath): + return (set_path / 'real_atom_types.npy').is_file() + class DataSets (object): """ diff --git a/deepmd/utils/data_system.py b/deepmd/utils/data_system.py index 656a7a2e7b..a3782a906e 100644 --- a/deepmd/utils/data_system.py +++ b/deepmd/utils/data_system.py @@ -84,6 +84,17 @@ def __init__ (self, modifier = modifier, trn_all_set = trn_all_set )) + # check mix_type format + error_format_msg = "if one of the system is of mixed_type format, " \ + "then all of the systems should be of mixed_type format!" + if self.data_systems[0].mixed_type: + for data_sys in self.data_systems[1:]: + assert data_sys.mixed_type, error_format_msg + self.mixed_type = True + else: + for data_sys in self.data_systems[1:]: + assert not data_sys.mixed_type, error_format_msg + self.mixed_type = False # batch size self.batch_size = batch_size if isinstance(self.batch_size, int): diff --git a/deepmd/utils/network.py b/deepmd/utils/network.py index befd571f24..22542adfba 100644 --- a/deepmd/utils/network.py +++ b/deepmd/utils/network.py @@ -204,7 +204,10 @@ def embedding_net(xx, xx = tf.cast(xx, get_precision(mixed_prec['compute_prec'])) w = tf.cast(w, get_precision(mixed_prec['compute_prec'])) b = tf.cast(b, get_precision(mixed_prec['compute_prec'])) - hidden = tf.reshape(activation_fn(tf.nn.bias_add(tf.matmul(xx, w), b)), [-1, outputs_size[ii]]) + if activation_fn is not None: + hidden = tf.reshape(activation_fn(tf.nn.bias_add(tf.matmul(xx, w), b)), [-1, outputs_size[ii]]) + else: + hidden = tf.reshape(tf.nn.bias_add(tf.matmul(xx, w), b), [-1, outputs_size[ii]]) if resnet_dt : idt_initializer = tf.random_normal_initializer( stddev=0.001, diff --git a/deepmd/utils/type_embed.py b/deepmd/utils/type_embed.py index 5d5b5c9888..fcd890e224 100644 --- a/deepmd/utils/type_embed.py +++ b/deepmd/utils/type_embed.py @@ -1,5 +1,5 @@ import numpy as np -from typing import Tuple, List +from typing import Tuple, List, Union from deepmd.env import tf from deepmd.utils.network import one_layer @@ -72,16 +72,19 @@ class TypeEmbedNet(): Random seed for initializing the network parameters. uniform_seed Only for the purpose of backward compatibility, retrieves the old behavior of using the random seed + padding + Concat the zero padding to the output, as the default embedding of empty type. """ def __init__( self, neuron: List[int]=[], resnet_dt: bool = False, - activation_function: str = 'tanh', + activation_function: Union[str, None] = 'tanh', precision: str = 'default', trainable: bool = True, seed: int = None, uniform_seed: bool = False, + padding: bool = False, )->None: """ Constructor @@ -94,6 +97,7 @@ def __init__( self.trainable = trainable self.uniform_seed = uniform_seed self.type_embedding_net_variables = None + self.padding = padding def build( @@ -134,10 +138,13 @@ def build( precision = self.filter_precision, resnet_dt = self.filter_resnet_dt, seed = self.seed, - trainable = self.trainable, + trainable = self.trainable, initial_variables = self.type_embedding_net_variables, uniform_seed = self.uniform_seed) - ebd_type = tf.reshape(ebd_type, [-1, self.neuron[-1]]) # nnei * neuron[-1] + ebd_type = tf.reshape(ebd_type, [-1, self.neuron[-1]]) # ntypes * neuron[-1] + if self.padding: + last_type = tf.cast(tf.zeros([1, self.neuron[-1]]), self.filter_precision) + ebd_type = tf.concat([ebd_type, last_type], 0) # (ntypes + 1) * neuron[-1] self.ebd_type = tf.identity(ebd_type, name ='t_typeebd') return self.ebd_type diff --git a/doc/data/data-conv.md b/doc/data/data-conv.md index d3c0632464..ee98ecf556 100644 --- a/doc/data/data-conv.md +++ b/doc/data/data-conv.md @@ -30,6 +30,8 @@ O H ``` The type `0` is named by `"O"` and the type `1` is named by `"H"`. +For training models with descriptor `se_atten`, a [new system format](../model/train-se-atten.md#data-format) is supported to put together the frame-sparse systems with the same atom number. + ## HDF5 format A system with the HDF5 format has the same strucutre as the Numpy format, but in a HDF5 file, a system is organized as an [HDF5 group](https://docs.h5py.org/en/stable/high/group.html). The file name of a Numpy file is the key in a HDF5 file, and the data is the value to the key. One need to use `#` in a DP path to divide the path to the HDF5 file and the HDF5 key: diff --git a/doc/data/system.md b/doc/data/system.md index afaa5c3d17..d94c325cf5 100644 --- a/doc/data/system.md +++ b/doc/data/system.md @@ -1,6 +1,6 @@ # System -DeePMD-kit takes a **system** as data structure. A snapshot of a system is called a **frame**. A system may contain multiple frames with the same atom types and numbers, i.e. the same formula (like `H2O`). To contains data with different formula, one need to divide data into multiple systems. +DeePMD-kit takes a **system** as data structure. A snapshot of a system is called a **frame**. A system may contain multiple frames with the same atom types and numbers, i.e. the same formula (like `H2O`). To contains data with different formula, one usually need to divide data into multiple systems, which may sometimes result in sparse-frame systems. See a [new system format](../model/train-se-atten.md#data-format) to further combine different systems with the same atom numbers, when training with descriptor `se_atten`. A system should contain system properties, input frame properties, and labeled frame properties. The system property contains the following property: diff --git a/doc/images/model_se_atten.png b/doc/images/model_se_atten.png new file mode 100644 index 0000000000000000000000000000000000000000..c7fbb6d9c449406c69601851e5b4b4695b50088c GIT binary patch literal 99657 zcmeFYcUV-*vnaX+dCG4!JE`_3b zkAfL3|A3$T0eiSQx}s=0F6m7joiE`D6f9zA>vq}JWxCWD$J|~^19jI$U6g6c*Ng@DLzXSlXvA=P2?*RZm2mq@393MMB{-F#eDo3}l z0D#?G0KnA)fLosd;JU$YMgQjg`&;~nXT5{sp+r5W1M1Hbum#M4JAgc3510b%D2NN- z05}1@i*Y~-pdh?Odg~?yDKRP4Z3-F&31$XbS_TyX5s(B}{mElPb$vZc*I0i`yD$em z{bwc5!xB?-a&ydlt6D13n`1L`(k_Ld-KL^qxX18_nfXzgwZ3)Q|8ctb2oPMu2*c*b zK)VN^6QE%bpk1^A6sXUKfp*!k|4L}+7?@YCqUscG0_fT5k2Ii!lHP zRf$f3K>&yYC;zLT(jTq-{~!Qda{%PY6+H}3Oi1U_BR`FdHT zEcpKUYY19qoU!-qehD7u!YwIp#;Xxpe#;IrKz~5$)n;Q-Pe^>j5Hdf_Y7O7nJV*}2*^LDk=?2-Ek#-{}5 z9S6<-Z@<`%!<=@(5wXj)BM(3AS}Zf zAc%^VAc#s!ymR$oU(_8m@?ARAb(vFoa0K-FG^>5YE~@j%t&D?s~t_941J1YO`SKukK2^5N`5wAQe)hdXGkPN-*>Ba|R$O}&G933#H|C|NHe zB_#+BFJZA+t`kDQuhp{SEv9A1&)h7Uj1cvG(CM}<6ur!dyCwsHz`wuO{{TXpcMn*> ze|nnXy{I}k1RjLlR6vO4yJ4vQ4|so)P|N<+d%H7p3P^j8nprGLncR4<{7*z+t6%~= zDs-`-E$ni7?f`n=FD5~s2bPbfWE1-hCJl`>6H}qW6~myrS@9ce4joNWCV!fVOO8mw zhrXXL-^9NYI_bTK(t&?s@!#UV{j*Fd^t(eiuAQCHh#$x~Sp(?5dcD-*8`?IsJAcsM zJt9OQ|3CnMmvKcUoZTZzm-B-%4U&?PBqCfkqL1#(k&@STnm=Gg0D~qjVxI?# zMK+z|$G+aLs&6~aNDJp_n7^|?-(bPzWF;PRH6?Iq(8g*=xkIN`xyhP4G*m`dN_XE) zQVpu%0(lM(evqpx*dSC{5S;j_&1`r++4lBC8vqsgn0v9`mfu&rsqVyl+`#NCnCVo=EMy2tW5Ib38pMTQ3_ch{I|apT#&|BQ-)x_%Uob z>nNMYO>#_BXkB^4%%u83ZV)UBSE%_@ox#Hy^3BvvMf;6N?Bjj@YCW*F(AO0uIiD~R zeei>v>6sz-SnCocA(M^pJvDX1((@%5$UB{0Zs+mZb`~~NGaoqgyUdl`B@KomZ>s6p zL9}e`Lt`~`J*|p(KGyuz=g=0O+9ua|7-K6dbE>r(Oh;<&3XAW*s_<94L0tpf`AK)Z zw?l`rSAnI_lW9diU|i?4gTfl8fkoxJ^#zcKuP}H%*0d;PQ-CD*)=v92bOCTt$K{?? z26PAaXun4cR9pa6CC6{~i^Vqz&&~L(E3dFr)p1Yua=}GJ60V^JrYhaS7qJoY$_Rlj zWtI%qB%Tedt9?6t$HA9bXq)R6*42@nzE!ImO9l#X7(ToW?|zC2#veB4Qp??ncxB@t zp|Oei88SHnhv%11!V{1My8xymzsW&KP=g+(f>Wd*{o%7aJ*zCn#c9#B11ziOK?_&k zXTWNmA?)D5pHKTAa}dH%dh(!K17{u6|3QmK70{Hqbe^{<;C6l(!B8JU#OdU@ut$b^`U#VWnP`$7cq{LkGJj^%`Uu+#*LlsjrWUHhK(`=@ES=TfbcL}kOs z4w5c_PW%3h9_8RJCbjNiZib+y_mq%gSX(+YK3kxd+s?m0F~Xb7$|z35B1S)VO4d$j zB71G8%gXZ2x*ECcyFyyBPFB&9`|tF61!BiLhb1|y>oYC@S)`8s0ew4nsrhIL^lWv;Qec)NA&_q;582+JW8K?O zCI3Y2x!2_DL^f&GVRyGpxX7lur!`Y$C-o+>mQI;e!fwIc?sLJUo4caz_Z3C_S9n?O zV7JxpH)`xeyVN|ly)27g_G3oRf55HZK!=^(_PAdhMx`jKgOd#7&#aa=N>d|8s2SbL z?$Vp6cZwM&Lp}8iTer6<5SDaf>@HkTDxN4wiIP7AVvaM3!=MpKRylV8#0*e!5iCAs z)z@3iM^=*8ne!~1?Dz>KL#*xkPl&&@6_~Pcm98^6+a97i51RWP+o+|Ls5D(&jq04> z(hG;^&YoQLx^08H^vzPKFpasss7+J{)5WImq{FROGJ=IGQ7{XYX;?=&F??@7%em6jdq$8bpAx_bfO>Q}=>_h!;|s19w24x-Z$W>kWqA#@ zL`gphZSyyqcLwk}O>~z?+sQb7J+y`~64cIhuua|^@T_zcGUm~8ZqngKM=)rJc zALB_4*JN=XfcwO0GlYrN`;zyMR)NjAb!xiQl0wQ@uG{2)8SovJP(uBRekClDBq?Se z6|Dl7(C|)r;vSO?f&G)EfTydjP;6#(pYpyt?!QTAH_TIbZk=TaG*Fj3qiQHTt{3a4 z^nTL|V_=?xS(&rv(l9uvgdWRf4sG9y3BMWpxSTtI5y}dUcd4_BYS6>+?st;Uk$rO* zzslzOy(hGjmc95=624sFw4At4-u{YuE=e*!fAyESGo_W72%~a zQ|z+)DCJ-BC^M^tIQc>>-j?yPeo{~rn0B$E?$OSPTW?p8ckQ5Rh>mrm2#P0!pQI)( zeluPYN~_tAIC9nk{88!43D0^9HbkY>V?3%($W=*^s=aiv<-DIR>d2KFd(r(|t+&yQ`G>oLJ?Y@V# zt>#Km>3NX>=lT~9aEiL`x6Ocno6r(qSmRNPoK0h{#S&weioIzVd+=72oGe6H-?MsG zurhu$3u(1)=?frz9G_tb3=3Su>-wRGwo4d^+4hxZFOb#iEvq_BlE%lR-@x!U!wqWP|}AT`^DIIZ|ic?69+W%)8;QHhm21=GAS zTNhYb{T55pMoEJA@!xZ50Fzz%?LTL_KbX-)^^YL4TUupJsmeeRhexjsAbwc zh+lELJ=_`BC;cE!r2U_P);6tp61Al$BgXCdB0Fi%F9E;E@OL@Aq| z6RAg6wi=*OKsa>`LDtpT;!qbgs#5!EJ3}{~$sgJv5?$^34(96M`GY2%W6iS) z;L&8PU{i+qM-5=XUvR~mr z)F3Ax)i@-0;fv*3zF!Nl(z!q_qs!MfGzT=|RPb5EXg{ZNdZgkGEu4t|<_^t%4OQ6N432l zFvv}bu57EF8x>Zr>5)qku743yU^i&9Rr+8QydjcPP8#O4EBrvuJTHF~UuVovtveS^ zcwj!8IHB$K$seh442d&$9KF5~jG!flXUn=ls%79j#kY_sPR&{e&U}D9nDx>DY$AlH-hcY%M-&OCSz&KFqCJaRlivt+__$l~Hu{uy zM7}MrMh>5n2Gd*U)iLSPN0iZW$(f%>hbC?z*u?&;8M;<#x4X&G*QRW8PV5hTFFsqoY;usg$XBC#SM?1Y~QG)q+x^ zzJhZfrz94ZdAqvpV?E@vBYfo_n`RX)QA{rhGPb$c?Fue8ddRGDSH-22DuyL!B&fMn zv+*FUPm7|1%t>|@kKVkY9Vi&ve4e7A5xS#tSD7pMeO7X2kE_7`ZX7tB+3uy!0`96? z5yPJJvf(E5c&jBYT-HEa-B4`zi|z*V5}vZt$-9-!*rKw~4@Iviq3SBFbHO2RzJ;x- zf+w{eZ_keJ&;>Uc4MV&_BU~bkX-z|wGjp9nX0E}n?jW116J9ROdUJF=wZ%ERR}zl@ zuy9Vcn)p5Uen=yaC<=kw%r`pOA~N+{u%;TO!o_Ag6wg+|bT1eGz5&c4*hWpU7FitW z*`y=#2nw@blCip<4pWcze7HBqLTc%);*lGtW^bUW@7X7^l$aPXqGE)2tR8U~-e*12 z6Cdh^aP>bx&Qy(v+DFN%hZ!2qm(R?^)C4dMLJ2icYxqu%L2+wQ^4gX5O}qC?Nc}?( zM^ArH`e(-no0AMZRrWG(!ZdZzKW4{h4KX#UDU~bTcbBY8?BzVJCtn4H3t1V|K4vFu zSL@3x;J5OlRnHV0s}@B(2j4`lIHiXR=`u+<`*^tM^>GNZOisdGlyzqZJ{lCppPU~c zWvYqhqdjM1dT$W9>F}BnWp>63Rzwn(iIH~>xD=^gnzaAcpi!3(A=SipOx6X3W&CBi zj?6-_bnl9^iDmt5sD5=_xlp=DLX?<)C(q|JJP!dWD;INcXv^yXJ4ds{sVGWgKJfi9 z!R%S7QqOzu$k@|i)t+6OF*^(xSEO-BK7QnZVmAwr-R22H6`J{AIv7_zaPV1?(O%wA zF2rdQTiL7FA3uJ9UcF&rV*9K5W#Py(=ym-DPm1X}C6?zL#+W8aJU=Mvc%b?_46X@p z1ZV*j&r>z#RvkuTN1b{ssJ4Q9Feq9(}naS zhZ(^+j&C)s z_Dmcu%(?yKyo3!RTgNt^Y`Vi0x)Q&Z4Z{53Mhqo@GIP#{k~MyL|8FM)09f8wO(`!R zkr~CeX4;W`_1k>DM!9kT;Js9|;a8kok0AdXrRr{ETM`&=8Cn!^E=nHA+s?o|)f6c! ze6()?I0diypNo=h&Go-SYrkD&RmqNQ)b%xK`n<60JNBfyOEUy0W z_TkPlJb*gvBgGuqq{9?Elj_u)8xUx5YeFGCc9y$=Io#p1Q*TaG6$eHbzczBBf#NIv zmw+#`r!g^cFB*Mf+_mwk#nciFzu&44e!G3ibog6J9+tYsy#P0 zf44*$X8E6_8~moh(NBe2?nJTV{MrRz=6%|Bc!Sa)TD~gY%WT(;t8U-`0-=8;O{A^u zGTWzqCc*$I4vBSnH!lfUdl7Um)|eN$djas$590D_>gCvpo}GHgSGP-^Wah4~r7Ku5 ziZ~(WZDiw9#C#w~?$7<%bc1>-FX(g5KJ?eXA1HRIX+g!E5JlSHF^+Jqab{~574q35 zT0c=jXmxfK)q;pS0+GpI5VE3D7s1`LXInQk%AVW+nR8Rr5VVKc*d}empUWvxSXX?n z%iEaPJ*txNR$#8umaJ)a_OcyTu{CJ%+%_C#8#Q#f0E%@Yzto>8hwL8=nXNzfDVn=K z?!`SgD0bc>gSZ75%S_&Q^ZDz|rk90-#4JYI>azVg1a^7Qd#?qgA}fx{rTT+~QrmoL zd|8%-FEicSNUc9gxOfH`=})X4^JuP?UGJDV>t3*G-bh1Dw$eHyprcSbzpe({-B5#A zrqdx zqaLHIvKV6dsiXpdJYDpBs3hLV%THpg(FS@}IYvd+>=>F;F3NAudsjB=7@E||(3u)V-*mLsNbNIM)@!aIl(J$iqa5&DJfT%s zjtf_4dwnFAHwMZIRdw;5-#fe=`1tLAG)`hw!WHGav#vxi;e5y%38vLTor=K)aGgi6 zg0MGUOi%9N``_cxuRA)*7Mc7Dcf~Loy_GM4E5dGfS|^QbB?Y=}bj2{MqgeoyH7R>( zVCqm?aLQM)D+f3@41jANo9{g5P{u}6#Reqk(15p?Klev&{wT&ZPT=Bfe9N%LM3=b9 z@esX+j8%f*IfpJL%0|6zmT}}Tz`(GG{Wx2TvTJJBosG=nLgCL037%WCuW1g>%ST{q z*mOhqkVM)Heb0R7Lq7n-H(H|!?)<6p<=)93>%?DGcL^ouQOm4Ut*TtMv0iKZZ>dNag zM5RPEuI#2+NGYwHmqYoCosJV9Z2$HhFtHQ_Rdsd9f+?d-3^51wl*BJ*7HGJ6Ir9UG zNhS}u^b4ri7i_^+bI!iF5&%FgXLWd#SvX1mYAsXcVPu=}US?AN>Jh=M zoSNpEq1ydNX}ktAamY#nhwGOO@QypIk6aFaw9J|2He}H%1+(7ju~09&JWu*rw-(`; zcq74(0H$rwB2DZO=^pXs%x4eBWe2xZPKPev8^^B}#IBsSQ-m@PwzI3oCTCYN7FBgq z!(xl=L!0O~whhJeMn4HR{l!icc9$`?np|&I#RKn!>)b8IUPq@mtF|KEnw$hhHa3ES zf-~LYJ+{e~bY}JJ1-`l2cLHst4(k^{{*Tsgu(@}(In`&;ra#rT!~ z&AtMQswrbmzMgQrpXNs^kKM^{I|5|%-0?Ay_HA@RS|`kUF;;5+XDtv@vbDe0mX zlTLfpNP>r?6&SbAD`>&}LI*_oo2yEpmbS%p5LtJvpXquGZ4&ZgYyb z8eo3BqyId5?E-i%K1yBt(p#u-Xf!{>Z{rP<$o91T_3XGp+E<5SuXth$J<}#mQi2qi z6>-hU5ZKXH9k}MX@Va=B#H5!|OIU}L>?HT!UYm@tDyeZ$Df!$R6j&{8K^WlV*>Bd6 zlM(*Nj=VaO|MM8c$fGaq`LY#Qtt1+;ibyHlS(^j{7#V>mcT=ELNOX&ybRehq)AK?n z+Wxfm^Wf<|9F0erxzD}4V)l6F(emk5P#qk&T2_;&sHBSr4^WIqUvtTc`geKUp%ZV%f}7u-M46xi=qMQgnx`|P5#>u8 z>S2K3^CjXk;fDZt%6&68?rwLi3G6pQ%Z|L}*V%r=k^Ln2k)!QiyVCJT`Eq=0GaO)? zpNb;m&2dk-fBL#vfZ>X?N%E~{&+XJuLnA6gK^{{RHQ@a{0HI+9{v`q}t`Uj*sQBBo z^>5VW;D`-8AY+vU!U&#!A-e{>)@+MI`s5v@&b=?RSELJR-nb6VTRz9m9@mtQwHV<6 z%Zo`5Y?#QZ2}-N}76M>;Mxz!Te-(cMzRz%e><(ixG0gtc$%dMXv)lftV-hp(Qh8^| z9tTjkSen-vk#dWj_)Dvd2XN)ehdmkkK6)V4kfy0hs>ZLgl{+sy`fl9V$7pgOlDHj& zGgDA)QLUrnV;x51ey+AXvv)rO?p^>nu)cnj$?Z!x)E-}s8s?X6?!dO0t`HlD=g z03WQcEgH|O(fakP3{tdCcqQVSpBmi0F{B~i+!3KW6O%1cl%AJ5vSm4hhpLg_#U1*- za!jFPvGl6@W2OHNVP|%n(4=$)GQXW!sZ$PT{?s)8Y)o9ke&~=G<@#QS5qHWgbQWb3 zC%ouM)izeqiNv;8b7&Vzo6J0*33C&#lUsEj&p!>fPvxrI2(?K_v@*aI3bQS(`pVUx zs>i=GyZlq2+7Vpidxd`vukj6Y`m0~R`f@x~cJmj}{?(kfTX|I_`-b_E*$Qk5>#{SfZeHf4o|GGPx)oyyeN%)`8S(Y(ffU)CHWF6F?3@H zExw&nKl%M6af`+rj&Jji3t)nqanUc@R|I6LqP*szQz(w7-b!iZ#u`cTmOeQ_hYnre zJ`DTMJ#pzI-A`Pwz;;@7U9cqATXs@xSqyUnzmm;ZnY>|NFeytj(U7X#PJ96FYnerC#R70ff zD?Xgb9s0fInh9>Nu$2YWjam12 zPPMLNYe|l_JC%;#7ST*gkQ`aIM?PK8nN(i+@?oj`xsl>sKNdE-SZnvmDlHgQV?m4V zHnDZ3ww|J9wN~hGw^R8mtmGXwHq3RTB!V?G%*L%|eKAd(Ag3Rk8}edEp(}k|29E2B z*kSgz%to#`Y|)3fj|e`1G9f5GD5n{iRel&8WnOBmW+?Sx^R#SjpdcTS&B&jC)OP8G zInGXfmGQx{cwxi2T5U=! z471MnKyB{+pmO;Yg4V@_y1E6psFGzS%)4OA;{j~$bzAd{IB7R1OHs?$l1hx*xx{zj znmNO6r>+g}txd)h>lB%!54(mASm7K(h@GyGFs6iJ^n) zwaVn&9^TKi)II9@lio?n;CbhPc0WxWnY^mrvEf0%YOuAi>F3o!D|(qj{f&uDtIBF@ z5tA8|sT-i~KbFQf1Tdj}+X)rB zB!w5+4)b*h!Vzx>2jo6eLqNe3_j^H-JHeAYrr5R$EFs&1JOyj}AIdzsuoamO^mh0Q zl+S4M8{#*{>1PwOSY8Twj@+-3Wa9h8rZqH4$~>PH)2<5b-BBEhb=+xl5!febwmKlQ znsxD1`PgKsK=Ap&rmiBF*TO8To}SYB@YI#{PT7C}+Aoo1jdE|_ucyF*AsVr!IQw!X zSnp=`49TZjXXa76_oF>}jNjE2hi22HLd#~021acMtk>un(uiZA%L-lQ*Az{o+nth$ z2i2BObk*mx-j^nu(%k|d7zj$%<_q}fZu5$4P{p2e4apJ)Q$!#kb1fhDB?v&~(b^6K zd%grRR)b$NOK#=S3mv7kgYn=%^s+0>fe(gL)%_mmpN+9PK_j^p^Q-(CiZV3p^Ca0hSDL94 z=bv5x@@^MEGtGLWVYuq$h8O3c5B^bjnCsWprZPoJV(m5n9KqL-PV)Er{x z(p`Dd^!`US@h=YCmj4aG|4bxF`6Ob#k9*tZJhpK_&UfpT2RkTc8#SXCH^oE9SEueu zInE@^jb8PSGS=C@esXdFJgD*Y^5?l?Du9TMdQxhh5{R<6S7tiWJ)Mb*d~^XhiJul2 zm19vH_$XZfAqOT&+hy}VxkscWvPP1vxuxp_REDB z=5#V~ENY7JTrv}L8lCa)nZ+*#0QhrH%G{wELxaVyh9dB<>Yu>n`i$UtD;mmm__|rm zkRS}T?o!o&bFJid^kK3>QIiTNsidPicc0jm?g9>F;Ch%>gs40qN6`8f(`lJ|{vrCU z*>+Xwz%Fu(YXF(F2{xwbqS!Xwuh7G2fFSMeTRMWMSj?!|H=WdX9s+0v^kud3NB0qw z4Y`&cqa%n@Ha}0oqNI}WUJ3e(~}09n^Wmh?E&gVvR8HL~M6f#>A>S z2Cp~fin>V(;p?-6IUn6Do68FI^*STyZ~h`>b$^P^MG~YY*rgF2W?soyPvla@iImlF zR*kqh0Gs*Y2i}p1;AP1lY`25ks&_lh>W_12}}@!1+z;XFucD&ME;!gF=j+yh^r(7M7L z&Z(5XV4d$Vid9-G?1SUdZTLkjG94}%aYKG$=jqce_ zZ{5C1zqWvyu=_!xwyt~kyFu|mI_%x z;2R+@l&4*HJ1pnu{wZnHP=>Q>ktxeFkg}M4^2zX)+bBiOj8drklRXgD%g-amenTr- z6_r=wQu^oGrq>Qm!+5=|Rtq*KlW1eV63)tddwsKJ^^eKTx4dPq)Hau3OzR^Wld7-R zb+YL$`Z?k&wv+7OWQ4#+QX3Zh*1bBReC3-Hr4*EzI7ju3;!@DnA{eJa=L63T&rVNS z)#qXA@~m`1!Dk!;-rc6Na|_{V|q+ap6Nly)v%0UkuMg^Hret?Q1e(b4kkPt#=6vI@V$>+FYX z9z*tp=x6riteP3jH7R_pUP7M zQM~fHSmIcDIbWTtse*nFa*OboR+kJjC0mO}Yb&vNV2-j56_ylF<0^|qiekc2-psNE z)M}_m@9jf`tv2k8a(sO1)W=`JpN&7tWzJ+F(o7RAVf+;C>M&xQt!k~AHIc>;jrSf+ zj|M#OVD8u~N`xd`K&l-2;R>SH-4ny+@}8lnm3dz@t#z*h_Bg;#SWlox=` zzNe*Fl!f;?-fJ;@4a?nkpR`9)Ahi{#xAmI+E~ydtEfE2QY`iw{8EvT0LQ#sU(iIZZ zoKJP2vqf`T_aUmmAtm;DIaYBz!Sl*}?{R}l5}s72ij^|go=s)71kDHmg)d-5?>EX^ z5W^#r&A5T2Q;347@tctwAZ0HEsLCK=!+19w@vY?e#^TgifHb$mhIj_wr=&|XF5+9{ z@ePGesqse6KFugCezn9S8|O%3O5 zuM6PBjo>NDU7%>I1@GAnva%zcv&ZVj4U4 z5D>ldFMr^rsYPd4SDbfrCp{Alpo}>FPIydBV_|*3v%;s98P`8?X3k2H!f9XR^L{UY z%trjX?FE2}D#X}H%@NxxFIMbaI=h;X>@E8irn=L$+{YI&jEZKCb}k6O4aM>hlRm^X zjddTfwKWtFd)bRQ=4IFmv)dw}sfegqsE$F*&TaAl6plAqq9nKHS#-aB03re}HTse8 zL<6p9o|99(;(yAzVs^sQQxL!U{v^vd-SMAcIRvrF5(Kl{^VH5YXm=uLx6mJukx3}4 zkxR$Mo?z;AvRl|IOC<{MW{q*69Kj!Pa0ulwP2OHc5&bU{SM~U_{7aXx#D7P2^C9}R z<|hEk_u=&oQBCo7mN;RIfd5Q}rU?$5!Nxw>K^wqkkq@5-&nY zbVpg#afYv64RxY0+>(f6I&aI+qX(?8$9opsu(rWA{$x*GsTPcVm0>i!iJ3`$!(AML8?f^LO`>Q2kK)^#X?kh%|%x1F6I zob0)NNl#dlwTW4^1Ur|7RyFjxEDce)9xri*e9b5nC3+z9E_H^lAu zw5)s$b?k%M2cTUo%F03u5`V# znqIAP4uXa%)q_Nwr+u_pmFDwD9yzE`EU8C6W$a?HrzG@E6~QHS6P4Lq7d?=&@dr^; zTNVcL)TfFWI2D4;-`dw~SXqML5rU9D`|s&_eb*D*n&7W!>i0hmID6?^J%G{X%@{jY z*-uw{o-KM?Ns5OK#*ZZ4PqHa_w*D+LrY$_K{UtqbqYxC|gVOT5NqwnBfvS5;fG;G? zGiH2yPEgdBHt}gX%wP=SAXcFfpq4$BQZTUooomc<298BFTw~3yDUz#sTh+@UlG1DX z8|Oz{x)SPT4biJPuFCH1YD&43G9pK;E~0+yla=f36OQnEipX-_c~GX$<8c~@+|kHM zaut|jJ_DJ3zr2bCgr#HaZ$&d_yy^uh>QdHf@0ij*pvxFbNa)zL93P!M*Eh)T(SrFE znpd$wjXXgtHiNR9WzJrn6Gvx{E~huF(eg~QMoJlDu*oVlqv4q**R)p{6mB+J+g0kS zeT982S6}g!xSA|NV+nP`p2Qo59L5Xt4Cyy`u?>vv1i#6&>2=Oj!|8-=_&8nwT1m^I zeP4U+9E#aQNGmo}HVu_iuxh`5X%IMu>1dc)=4o~vBR@?|6u2duBsCrt3|(Fr zYIOQE$$zardvBZnRW)s>4!_#~)XiX`B`nrt%JalGXg#8s?t2dpeJz3|2-2<;A*zvV zwdFIaS@ndsjdgST0yvT?gWE?Z>6Fkn`WSHxuvb})SeALPKb$Ea6;7aY-65El+Ajz& ziGb{T^!uf>?SzxacecUV+4folDyz;x8(*w9RcfjlH+n$ksSQFQ4k2y^*ty&FDUc$P z_%1h+(cEP9YCJVaFO85ZSJ(QaoFCDV8L5>XyIjYo zbDKTdB}N6CUH+CogB$6mj=};Z5t*7!v8RobeXd`Z7FFmwMu?r26BQ^r4oi(<`Z)Yc zaBjD#m?x~ThF6Nsva_-7bTubrP1fPdTT-jA&;&@1y~z`rqYCLn+yG%BHP6`^hRnc~ zCx;$2@-xTc_lDhV*CSASz}a~%x<&}&m5<9g$4t!YUVThdbi;|0j^mj~;Xs?pV zGWoWePKw^pZ2izUcT3?pGf(a;0zSB0?mn1@Nqw>!HBtR}m_pR0I`)F>RgsB?(1=Xfr$ z-D=s!CqC`bBr~?8cyTA4dZ-(zY!cC3iurQ7y|kJtC?RrrfkNMB2QGF63awVqmT$k= z;^{g6pcK2^=7qW;ELk&OOEzQSoupC4y;Bmwx$Y@h$bj~Iw7sCGuAyi6ZRde+2BP9G z-HRLLcbeC;gQDg%t0#po0BwYNB*c)*hiT!yx38aom+(Vib#V!)p^=cKA6u7K&(P_W z48bLK|K`W;q@WDmbXtnZt%xh{*8_97BoBWjn|@m8pimeZ=U^=lefq-0aYeI%%MZE4 zOiNYVrIZVztwB>H%{#40U@~nbm#ka`iP3jy?_nCEDc&i9s@i4K2lqQ)6B)`a+$naG ztjSZ;c8VnZxH;I5D?)jwM+fLa2^?Y0X{T^2YksMvVp3D<>&<+U;X=2X>@Vc~SA;APF3jF6xB`Wibe&hnc z3@Af=^*{D(DBc72ZmTIZKAcUSm>(D7+h9Z&}X;bllGD`bn^P`=kuN&Q|d zbUl>FlEFSg<+@5TzHV{lr`2q5C}(twq_(}d6K8a+X2ve0%RG~T{C+(?ci9p%TeXx8 zB7YJ0sCEC9^dZ%DbgYIEUGj;ZOXSyx7Zr7pSTWRsPk%&KMOptNR2UV1jD)*G+Rh1K zg7yO~3ooo#+n{+1>xai!G{H)wD^w;CqF98?4`@6;5^`~H1e%3K1CceCCmCf$IssYi zPpD@^!$BR+yrcfr%;wrezxVpEI>xQE)CZdM~>z?=r?ejm5{ukQxy_1`#*OUFr~ z!Ny1A-?)E5*H8yHQS%*wM>qT3UpK3Uosy@AO1CSE-+j`ByqT(l{-^@06hAiAK`5s?l$;Y9$~P3&gg+9}p=%ZtoO`&;8|ux~;_GrFe)^ z>a)J|R~{J!;xlp&*4wSc>^;2n0bH)%l0Bc3-jld2{CVK!jf^6s(`c3I#`6uHTsOQI zuve1P_7Dx&s|S-(>5lwLBGBOtQh&q1lVs}C9#o=2H(Yw>GR{I_r+QW0n2cm$0taI&xR6m8P|76E9Cy*i;`%YXUmGhMx4o>$!Dp>`kE!sf~ADjq4mfJJRu+E zRSO8RSJ5RV6pjiu9Z`H=u4L3PT5B2Xi!VVML>a1LZI??;cS!2?28q4XI@pbLw;P$f zLA;4~16z(}ozz0VsG#?_R)?y#ucCHGl*5R?FjB=v*PcOt9?hxUbU?&@C}nKY)`u%N zom|eZ+(ra1wQn8FZ~3LD(6^|aGV22P{D~%8<2$DJg7xRwm`T6zBN3|H`x$t3K7B(8 z;`glyu-G$d6*?70AhbAnIm5<@gWa>kh|XoDv^&AD?iT6r@swcurO{q{nM zC!fa|lB=7kstb*B<(PDs@^Tg`{i`_77nLI=MzqXbYqglaDVn(i*K!X8C1)gshWg`w zd_dxZdjw_N4DG(P8bMELi=;&yBu;jccR~Q1_uA~$f{Zm=M#F+<3z4+t7r;XwxrLje zj{jVG2l3YOoXVqJhkmKjYoW|W&c^WW&?Jl8 zZ=C0@eAWS`G)rkWao6v~c)>kCG)i0ZR9mx*3VkR~({7FU zx=M3xGkrTUn4!{Z?#`OCHmjmrt@&{Df_e$8+jnEpy{jfqz8$rjgR}o3d09ox z(9|*{w2Oww7*ZBaD$J~2z!^lB)S5sG)q+jsAu|U>)ToeR1Sr+`BPhL}9P@?sO`m6S zABpFS{zUU;EfFN1(jc=-Qz{k6Vm(;Rd1I|mzh+-`g?r>J)_Euru2F%cI4 zQEmJm@!3&N0gM>cJ$oTqTE(;VuwMS^{!cb!S30hAw5>NGfh)GCwRQ1JVqv!EFde== zK^r$192G1j`7)rQyw#XSWy5~fXz?XXu~1YxTAn_rmKMyp#gMc-nRC=&MjHKkUwPo{ zM7+aW!p;+(w49RteLvu>H-nY`8O^-t9X_W>{(fP_!{rO$&R>MKbJPwQ?g?A{=TfQS zVdl{IghASOaQBVe3a=smx8QGzXP8A3u94CkpsCE$7r)r=uAo)b{2w>);Fm)Cfurpr zomM`IHoY(b_$Px8O)CiT+oISgC%{j8YYe~J^k_{_6hKRm0I-sZ!F@I>?8YVU9_~l- zSDt-)a{)|>a%&|{8odft`{JatMY|`gG&6`empee;o>#3GybTrj=c*nJo^fE`NJSs> zVXkm!<2GpOBl?eC7f|A_#Fe{kp8-HcW=!<+rS#82}I>{*9+MlkqZE46Kt*d{2we!``0 zh_THzAn`8|sN*s}BBqNLs_;~r&5peonD|bOR5FJV7SZs97fah-^@kIY#z%WYMah`V zlV(rq^=04je{S(Ft6YkS5L@eyLyj-st7@a4eghmfn*5edb4O|$UE)XSx=NyR%p&>a zVR)BI1X@b9Z1$RyL$s_|jOl_tJc&JAB<+)*0aV{RMdiA68|${`wrGziboH+7`!yH6 zu_+=)8*BXqeOO-OrnO?^zK|bke;vLG)^1FZSk%u4p}v7eXFoYpn*xX_=F!R>ROkr~ zFC-y5UrEZxxt;q(%udP_tZ5OoftG8-OT|NX%yKILIa8?iwdP4}UXKOW%!^b#JwM^2 z?RXzIf9|?Noztl!ny}5Yp$mZj9FLi9D|su%o--!wUVXjfaK3|D0i02hFhucLsuwC~ zVO%NCTgTEZD{X-urkCn@hI;{!Pkt{vcQ_@&T+*3s3uyQ7U;W`{x;kV7j?4G(J>_y5 z*RK*ls5Ixcaq+l7J~KV8og)kBql|DvbOyD1dsD}NP^V|qR@BeZ4n!Jw3f*;te3k^i zch}DyHEny-)h+mIEt+Pj^?%OK$+uN% z($^C#c0^jws zDW7l|Xw|z|sI8Rqt!T!@l8x&ENYe{17`?v`4^i7zvV7(>U!pjW{at)j^irQ}G_gl9 zb|eLQdL@+JDOk6*k25RQ=DNNP|I1B&&>ODG18LEdAV$R%&4sIv&mR-HP1H7v#|Wdj z*LfkqE6mEV*;NMS@EHCGi1T5z8u*@*Ki>CZ+D@;V(2s>7yuX-5-QHkZh@>$J5bw1z*iDI)hmz5$J<0p*JgVad4<_i ztVufCW-7W3=Qjo(+f-1B#a;mB9;hV%#z5LS6t&qn-G>M@CQ+VtJkdCdn=(GdR|qJp zLxplZ+nx2*6J~?Ei7eeVa_U}M|GM=#eWJaiC4xBlIPt#_KS2>c>r^`1_+0$yi~Wx6 zaQk+O)mhSWp0}9QxD2tTH};o0`E}H z?i1W(7L7>#Rs9r@7mH@~NdaZjO!pMWj(bVX0MVzmccb^&^dh3Kj~cRkD~|Uj>h>Pr zzpae>zZ9ox{zEVhytwJIt?Qp|mXB8HEw{$5`ZpERytb_w3cUIwR?~H8v;Rq@PlfJ` zk833D-=Wn@y!yg^Pt%P)k3MhlE#V{( zbj^Mi;i1?)(VoyrKX_+1<`ZJsX0n76_1~V02wkcZ)?VKT-FXyK4e2Y7@Dlx0>F&pC zZPzF`X~8}3FbU$Cu(PD0iy6OTGYR5Y)1B^YuM~M#21E4LFqL}=v_cs zLJvLk5PImnc-}_Ox%Zym|J~mk?~U=+80@ja-h1u7)?9Ob-!}iqmpi=8p+v(2VYemns@R}WQMcVp^nB3O$$Cv*#I0O{+d6T|Un7>gWIwCn z)1ny7!s6<(^OZ7!AUeLXbItx&16D-i17WKx9Okq-q8tf!P$5lV*jWuHM=zpm?2pFj z2yA$PMJ`Xfli45}+h&WJs1CB^o6aA~F25#jRm3QJ(8YzHJ?hT|v)5sAn#q3Jo3F&d z!*VToc*E0`Q*6M0lsh#b=krY6ZIU1D)xO*vrk{Cy`34|C@S*(yl_uQiHufmvGPfdR z#&JDtOEys1b}&O~h`~I4zp_TC^7?2TIrrXMo$AdP-1@q*xSugHL2^mJC4!)(1vB}E z6Yzt5dd5GY&Nk|NZ(PwQN}wR06&X|V%nHh*UYY_$3j^KlbXPjr^$T_&f~5R`$0WT< z1i@@oF#}0p?>Z@ui((pb&Q}*p@KuXJCVFwVSLAXcyM@@Y2Vj3+z(oY&IbpUUiO${~ zqlHMVTxq5I(KcjcJy;~`U^3>nE!#4dPU1pNn1~`xEwiFpyN(v~`u%+h{EoLcJNX7w zxRq{rmQHhhs;~dn;(-;7Fes=VvHu=!zdAP^%kSRsVHWE}4luOPS~l==x>tOyD4*P3 z)btr%l|`A4t+&<$C>;5CMV{44Nis(>TmGOqu@g}-Zt(c7i1sZ0;Ko1i^yxVR2*WY; zeW778&TK|fkH3z*mKcsb9j#wclAt%B zpZ~E705p66%wwqp`;lT5`|-r7?IV37&!#omV^$&YKS4J24dSmq$;j|B41fcK!7hGg zF_-x`_NbdB0Y1YZ&BDv8?(;AIYVfqFjkEWE{`%+W49)3(sQ##+sJ3XaGe&n_$|>2Y zo^!KHe)M~ZYH9LTlGk>Z&P>9bx)WFGiMp}Gr+{%d& zKe6N0)h@oHX4S~i!ClAxI>VneTT=Dig1%yw`Sd{!iwsl^*` z@%Hw8kc(H7J%oaOHE?FSe`~u`wK|``@Zc@-4p;mB)QThqz$?1)*cf*rN5g*;3_Ui^4vwfCqu(>He=3cBjqF(5O{uWM%UXIQ!BFohHI7wu}H}HFqK>rVjSi_!q zYe8G#%!zy1qDJnRgNlMl)A@=~_mrI8`U5Eq6~%TZpDls(vgTB@o{HFsq}Rl!>2sdtMVo2e)$Bh%#-4uZ=3cu(y7F{ zc`n_oE=8%!^0W5mR5s^x#7=c6(9Y#9yL8wg7v@Yq!B1Kl9vkK*i7jRlFSl2^DoOLm zh$XMUwJ{o+tn$`%?{c;mM~M`ae<-KiwAFa4tD6@Rs5fZDY->8Y(77bmmEs#O1hR8d#xQCnNYw@7p@MULI~A^snPbv&s&%aavwi6F zthfdU3_@_PGJagqN{e$h>bgNpO4KY~fu(3&xJB9 zH)f*X<}6bxyl#C4T`P<7lmYl){!vGs4AnvqDzH0Z*{vxKN?;Y0Cd|SsfXLgU08sz7PolL`&A7=YSKEzz} z9EI;z5a}E8BI_ZeVqib%OaN3-+bNj8Ei`Dm-Cm6j5$*)L9y0eYa;dRhEYwqAfFe_@ zUO4uv9&u|Yq0CKMdlo|m+y4Zank7mWlEZh_-$Y(w8Dl!J9K9RnnbFWpJ-ZI@nx*tN3t0uoCncgVL84_eV6 zUNDrb5PuFg8Z1K9+rcMZ*>a0|#xQTM^tjp7-;*OW#HYAQe7O+D!J}#lkkb8?yX?kT z%XOp(T@5 zD>IBpuW~h#_N3kOQyH$9VNj;{@_0E1UeFp)Uc;{(*RcpL87Po{&9nYpQgXgyHc!tV z-&M^OUaoAv`Eh8vJo`~95YXe#H1Lb)7a2>yPC?UB{3%D41@QJ%My66OqF-VqjLd7b z2{Et7DdinS`vInDO1Nyze3`z2K3YL%P%cP;G`|`A2gEd1+cUqLHFrz!%Ga&xPU8W5 zmbbFU=Xr_G6)eW)e}bxlKH^dH5&SOe;O)s+vbrAXnpQ!umu>c7fqZT*^m0p=TQ`Qn zP&jkZ#IeX#RUGH!-qBCV!?_H{DEKXzX4H`pw~Lo?5%F$Cy9RAvf0G#xi?7mmXw9wZ zJVSxXAE>p%?``-L1|+MG!?>_Mfm%$d7%!Zu7MJb#*{VqAMVRV?p}-sPZ~Ap~F6gY(>K;Z^ta_rYW7SjE^9IptylN0V;~R@rHze7K1hHjhVx8@rso162Ip&EWCm zjm5d{Lk~NS5B0W5i8l@qOZS8KfJ&C9e>g?6r z{5@fdPjJW4t-y$(xgkG?>s#0=x_^v|fus9s^o*ALvT|0(7_V_Wk2s8Z-Mgz+j=Ek; zhQmG9yvFBQVda^jD~b{j2mT0@Xh_?3fs8F*o?~s^psB^f4joOCk-qVZ%`Juz78mo~ zBvwYJ`F08#1@iWxZc`M7raiwWNY&P>`9IVPN*HC+F37X~vdo3>%AdAj*l3vds;}!g zsgk2qc_655E0|B--Y_2-Tv>CRv|<)1%ji-N)!4j}Q3npF*pelS-B}DwyOQT*aM@+n zRViDrrOz}qHf(stT)u~OJB!}ZBaL|z_YIMDK3+x9)0raqB2F)NZ~)vOE>I1Nv+*#j7=dGWOcU{}5b)4YCb&ysx57lpRos4MoyC6z1Fec;oty~u6NI43r{5dVd z`M9=ZoYP9aXZLR%x_UNFW(&H&)Mxt4s*-C$hgEu2 zRy5Nc&VCyP@lLKJW|34zBRj#WHGR*+^$@SZgqdfoj3*a{Tdu{rsq#f#_W3Gjwoa;& z-RSs6Tmk$e<+>zOlh8Hit2VP<^qrGUyIcMi6AN`CtnqVR^|if$7&%eNER}KNdf!{| zqklP3rnZ8k$9@4LoGOU_sZ;`BG`U_B9(W49RY|sHY+LQ?CPiQ9^-oRFRY4}0lQG*KKvnB`Z4F2|z9h33!@wZ`v5Y)_w~$Aqk??TH(%w-YKm5 z%%rA6T4vF@6Dux=0Sr-c(?aM>3tBx8!T3K-%IIaT_Kq<@ci}7$7<2l^+Yufm0&={v zv-|Bk)~ycKX|2g5bM81?>M+q@(A5EbaVj%T%c=@X_BK~YiO%ThQ^@031voFSy8#9=K6{c zI6*JM`S%c-KS7n<7(yaS(x#OiCn%rgIQ>q@J2Og8Y&x78GTH%77%B*V6Px#3lVQ$H zqjE4Nq2+Nz67*9YCs*9Aw8hP#!=XF|9`+f$S`4ac+DYiGr9@G~pPvPh z-CY~IB=tUnZUYcj1mf-(>c+k8c#Tr&90cBn_)VqQ>80hQicy)TiZa7^Cmp)Ygrx_m zv++=A2oMnm{lusId_T^eeSEgyTCZCL?_DALO>s@WS1}K6jIJO2PHFt|a>>t4el2d@ z1_a}Psw}G4zf|4B5PZ+W&Yb*cG9Gy~Ie9{Ft#O&*X2Y_Vhu31vexgm>n^C;7HR|^^ z@)T-!ejaz?(GT*Qxc_&1p>6VyUjF*=ZZP@^|BEN~4N-{q?l6uMkSDR5XgHE(R5-h)9tK+=8_kj#cz}{4If`Fy^9&WM>si+sY=sm{CbO zJ0!kf&-uhK-=+i8S56*@?d4~=9u}he{OiZkMD0DW1$#^ua zvSYNi8DqN@+)#zR=6cAl%ld8hTfiq~v)BVb<1jKgiw+v(s6e!Nk$kEp`0L6qdmOL) zV6$XVkWqL){Av3mi01yUT>D1xOrv+JJ=A) zCe!+3+<*9FqCGC@b$Fz!jc>7aIk5Ou!FoeZ#?QRjC zRgs-SCT}&!XrX(^p_5C}V~K8|>UEB-@iup4lF!TaB>QZ#8MNQrH? z_B~B|o*d(dHS-C!?9px_DCk47BbeDU=yek|v;?@UhVnoEc(M3ak|9gB7#VN&1iKXG z5PT*0kD_DolPilTEgBtlL5&I}DuBHS7R2N9 znQ?LB-ORb)x7H)+zE)x+Ak&2lW*Yrpljz38HMBIub$)@N@Jx%RDb)37>r(?lvwl zd)xviU7G3l*$OavdIkLQe+GJdM;!$QcIOJUxM%Y2(9od*q8`l`n&?##Jc_i}cvad< zn?Cd?eg;v=)UHHy3IevFl=&p&>f9N(Gj5{(kQ1e_3KN~sXzlR1_{R$+y|7{Y&8oYW zT&$1w;G5DNnMs=)1aBHY0nv!(3vVLiVquE4w6Kklab2TNbpc%8F_>Rx_Cy&v#IJt%uS3)oKfoG92U0BxV1p zADPL-O|T5a%nAO!tCV)E-{%XKV`F%s{9#s=YjW(ItKm2@(L4}hl9sx<7HGQ{)Ft&4 zwR1qVaRBgBFi^p+zIMavd_GnyM1tBtgMscd=4m&agLhP0embcmPDq_d#H@a<_h!m* zL3!2rINlUsklGX#Ocre!Sv1}T&}rzp&3mS)`ACJz8!IQWc_GXkW#yj=*4`|4#~1n? zQtfO@Vx#DUzs0*W)DVqPptZ8Ot+-M8|(^jAL!SOb4sWPEoYy?*c# z#0olh+DV=yc6C{C<6sitpdSa&{@&4Pa|dOem%b7cf&cPg0y8gE7NqCay3+k3$cH= z5F-v~gLDA_j904pYkb*z^sQM53#GZNim1ryq#Mvk26g3q z@sRvuowO--H`HbY()8*nMT!_QESBQajhdAsBV&w+x*)fj|C|52bm1K+*EbFIaKagBL zbbD+pp;^=h+#ETmkrObgPL7^6CdZ5jS84tbyuMbc#Fo$o8=Ph(+;G=7VUaw-Uato* z-VALD^?=wJZp_ob$`Q68n>t4wlADmM%6nzbzA!$Yi|Ka8g@hn3dsbilJGD9GQ!21R zxvrbL(2SHCO;39ktdnGyfF^6c$yA&+29eRF9%J7LKbT75;@x+#|IjLn)*~=m3+r*Q zs;BCp7OJ$>6^xk?HMr-3D;Z1VG3~3=;ai+wQ#(=wR_2b>2uly#K9}DeOZ6d3m$qHD z8`m)bNN2m+X0@l)M#<)|-ah}Y?+2*-aqcBAw!|zJs`y3df@yPtl99R9*Vw0L82{er zV!$tv@^dOP0tYv2{F_A%sQGeiWqdWjj+kmHKyWvO&sl$1=}Nm`{w}=n@zz42r1G2} zSNXB+(vT3xFi1}D{$Fif3<3yZkAMC64>}c~gm_w5m8Wl2^qtgbnr9)| zC0IZIF9{^Wop0y=Ng_G>`I1IkQwm*AR?v^@xrvdl-Ua=*sR{gS;$?221 zDpRf?R!DyC#}!T3t1!@I7Nz457+IXM^Xdwe3kKO_mX2qI!r!)XBOu-0faMIP2>);ClC zo4k2#;?mZ$8jLpYox6YB2C=_B15A8;UIN#48iU!vw*AoX=S1mn=A}8!RyI#(zOeM* zs1nIl6_9PW997DySHVBk&a(u%XwvD3aj%SJY2_c_g)Yvdbp4-nO(syy{AAUQC16Mvl+ELeJcqF6r%lx_Q#bR|`Eqq$ zg1u1uLhA^iLZD^gD{r1V*F_8h@6OK_e+GHF0le3o_$(g8@4FspT#?GhdvAyU)s9#i_s1fL6h2jS|Tk5FC<*)vSp z#H6OMLzX9}#0Im;rB31-_T#jXTQW_NIYMZ$)Y9i;) zc=AHUa#lwNl4~}5LvKI5l15+dWMOX)<159C7D`b5}K5KP5D)QXQ})MnavtQrN`zh#OJQ+$hAQy=hC7{-SZ=gm$PT$FG`4)XzT*C6DLAlD86rJAA8Aj!D z*N5F-3{i@vAIekaGmEZ(SmJ*j>}TGLzXzWGVA`Oc{#<{RZ9QttHfVwi?RzUEhzQx; zkaMZOf9fm&fgUA<;n;CnsI>uGh0DmhEZq%vWfpx|t;2P42D}N$Pdq!DLQJ}Mq?Zvy zy2%R8lnWJ{Fu0NvF|97{<66e-^Q;S_+Tr#I-E1son}v$de$^`tP_l{hiaPYhvxhhzUqeaoNbZH2gmm<3<9dAn9FO3KkITXeI!19xY)R?+y0VS!wTL-$@6oNuR6Z!fA zYp|oN?cIaJg?NvNTm9aJ(0~uJJB9tz!EJ_e3OYxj=_D$eh2OfjPc&C)qcr<}OuzW` zk7LUKi^4CZLTsx3nwkz}?Nu3l_UD0BfHm;zC}#lp6y`n1i!1h{J^*H99{*?f<8Cy% z)B(5*BOyB#S7f%QHv$e zwJjb;(;9W6q(>!GI8hQh!^fW-6EGXsRt2z#ADwE&N+7gx&fm7x*nLGdp@szZSw>5g_VI5;YKQhqvxw>ZqW8y&nVi`A1Bww9!3I|8R!rxM zK(K%=6GMLH%nu^c$9~LDHpuKS2gGdF$PAx@_!x0sYb_9TK4iJq6cZQI-mIG?7_I2g z*Ek!Sqt1m$5J#k4tW-&lljz7DzSmQ1N#qDwi{i5pXnvYhzP)yr4)v;DGEE8v=1L`}H0jH_l*Ox>!%To*`+ii;XtapH{8djj+-h(cHKmfGFEDSsPWL}Dx^2?^D_*8$R zRYp@&4riCfidi^JAWM6krPfx-LL7(yqI*=kjcwc9l|*<_ zt^ToW_^8Js==%Mi@8@qky$qT)QF+$)((Nv6gNg2WR^qH<6;5GO9@qK;x`I~ye99lV ztCLzZm^DCuFa^6;U5@OB&*751jj2q%m9_n6c97a+stskd-OeUd`mAE+`_T z2kXc&;1=JYZ{;uAT|`QemTK*|z{<243}{>VAnVAC&UDuS?mdJN{~FxoU}|DXuL5UJ zk-1Z`*`AkMT~>OyaTx0t7Pp)xo?e(AFCT88yd(UhGdL9~CmxTR?`e6G!7*+~eAk|z z>FFnDadn5K+2SlG#@z5v(BPB7MuD6r2RhD3T1-5Dj=~qGnZj2$k??r2-Dl`13QY$m zfknLl%>~r~J%FeAFr^~a&auI`=EhWe7#^z?k%n;t;F{F3?q*YcxMepOXCzSyHv$jl zbTi4!^KeP&AI-g^TdoY=x<%eCe}_-hvP_O@R(`E~uCzFWNNJl@6@WCLMgt71q=b}i zk9u&hwJEy@?*0>$7!8tBUqy#c|xGEgNkA!xlmyD|DmyA5EV|Z#D z(4FEi&yty$sof0;u*@on0G8Vr4~csvJk)ov8r8`fksmwP%4AKat^a?F-}^*>o+8HcR)icCZz9);RyfMpP=>d_liPL)+om4EtHDKSBs(i-^$ zbY1{0F*q%#)|FGyiRZ=*KyC!*J5~zawDDA3+9jdE>OR8!3&ms!0^^uvim~ZknC~A_ z8QFFtSt#Zk*UGfZ#uu&WBYdcDmSMHki5*?Orh!UdTcE37D#|9ksbp`Ry_yJ=B?O~bik z!jz};25S>EM~pMj1NPxR0BWF z$R?HK7nf+f0K<2e&W$LDcK%Qwly;`-2eHx_IK@BDnCo;4>n|IknKeE9q;AO;y7Jq( zr=#rGqlCXxo^xd&wo#t5O8R_G{!#SI%a6aB{vYc8r~mx-oOYw0G6@-{F;gsf@XQlh zD^wOIcV393a2z_6g}%y$uLdBOWe@)Z4cCwPc{>E}wgn&29lg;sWmV4d0-*8^?|@is zaK-lO@Sh;^$3H>4LVFib6|zy9WY6wmy7A|?B4hx`&(c3ZKc3=;MC(T^L>tJgc(U>K zlIG*>ADEr~n4_=WFG{5g%wwcGWwK<`QIdgUz4`tSIk1e#jieoz1ol>3+_|{JR2ijG zDx7DRs@DUEP;0mhUr(>9U8$6luMfu5q95`{R;<&FkR+%;_fUXE3Uy|!ZE`0W=Y-1M zMJEWq)=G98i(9rV)Va*NeUb;*wRbn+>r;Nt%8)YJQZ6Ze#kBy1Y#8jlYRl(IU-F## z>OqV$*M&6Ai?(8MI_H@XU*1;MYze4q4-VvnB3p!N;7H3jR#~apwL*10N2%t+x-Ju+ zgPW*uP3ZVao<@+6fX#e=6&Rcg56rlfALsK$)x~)yB^=BpKa(8n-6(ggW8WZawNu!q zlTDZyOnisIH(5MjOT#y)D`jUW01Bg7+_QrZDwdAq22*RjBRm?UUnp&DX3mR9pL96b3dYXQMI9!|3F2hI)NN?)24m8dhyQ{u%lCdN;qGPlj- z+ZB0a%Xg;yxG11~ixEhN(UVv!Oz(uHU|XwZSNDvBFn+5TkXo|r1)b6T>peR&`!^W5 z@?W(Ili|}e2>%iWMZ4K<^F{&&V!hxja?I_U;WCJZ83Pz&uTQZ&6ffo&?7%|tImk5l z0n)2;@g-T~gXQs$<1nePfMm zZ0y(+``q!}A)J8XM%#8*VBn<_mXG#fMiyeqHpZ*H24LBmXD8oL=r|#Fr;lece_axb z-d{);@Q_i@Kj|#Kj0$~b!RyLLY(rq4p4@8H6Z75iB8U^%^45yy{|0+GV*w6mpj<%( z+!8H^EfvinXZokf!JW_O8gF2DopD^Y$P5iVsUcHNQbSH1h2N%<-;tglPxc04u#ID8 z-zrh*zXsaCDxT}`esZFRDBvRcDJ`eAa6-T2^!}Zz)h zm!kcptDl|yp8c$8S9z4yNy|I5f7JPeLeUbQNCVq~x8Bu#Zr!vguN*yv%Z(n{Yu1is z@z{)niIJs7xV6UhS6s_P8YS>O%i76Np2bA2AK`76Et%--M!*JK6zA!24L-#s*Lk?} zzefBC+F@6@-c5Ue(UkD^T;1rjl-Q~^*h;1wMMcl;7S2lxmCCV6W+V^f?L{9XXVC<0 zDu|XWz5!S?{|&|r0?iGzdLhjXt^0AF_X3~4?^pIhHUFU1Q4|<8^mP&H89Q}B-YJ5M zrhs0tyK=EY&qTj|oZSZG+PrY8qj}NlUr` zvtqW!q`ZoCi*@%LhQFSsv`L9Z<1+z4uScorUx7~h253IlDMu;Mrph_O9Gvz!B zh>)oh0j#zZFr~vJy@y4+CnbtUiZ>yi{0XwFUuG_jjgwd|bVz8Kt%xx#L@)4_iw%`d zc~QLmdcYX7lNihvV_dF8t4MOg%2vt&pLOX-{}NZa zq4`Ehv~dmLu`w^+QZsBzMu*LQCWOLNYc^E&Sz--WfI);Zz?>6XS-t{UFupgktl|tO zU934#Q*-NH>?6Ub{hdv1=+v|gRj}sD8ftb0e_)+u=9}l6+VEUi7Mh%UUS&$Ev#vpB zsH3CH%r3Np+S=s^OYgi6V|vL9ceNsl5Q&P_^}@y7Y%T+?BapmH@g;;Vj%LO@5<~XL zg$jR2v>9ZTT%a&$qwKXY)#((NnFkwS_@SSfLD$WCLGU^vjWfn|_K>BxkB49R9xeA#a z%1~;xbnMJ`T^D2(FzfHeRc4opx6lxsOE&Su@2X)LUDuDot*piq$z3MW&jikEA{S2P zflaKB-t=a)m_a6N3zn4`eM~p3M!VoHp~v)Haq>Dt!@%wnho359>aJ&6LvC0R#vMr zd0-7@8T#Bjzo>0lfiL#t#WN_(l&<*XyJM$YM8ewhTC19KffEz?q}fT0Pd|zqgk!?7 zKK!-DB)V<}tLHrFc-)56Qp;|um8o?a4-e03u&)V#wq ze``Y_b*BF0u9~O900!;aJS{xsV09Q|Q}D{fq$I`SjnzYK`CL>b{byWMmxBV5~l#z}lNV+10w zA5XzXr{%2*NG}#TqHiZ^tVp^61+2-OoP>VRxzp8v_4n}@=mXBOZ_pmHy3cHSn{TMW zerx`rzQ=|>BWwgWGrEA_AE)k)7>};T*Sdjj>COLa6j&%-`_H_t|HwyEl2gGOvI#a-kKVYKoB`ZW1wF(lF|FT4W;Zo?g0af<Npvz8@X$J_J6Veu|%B1Hit2l6c;SPt5~(81i(<7sv8h&jS}X_t}3 zrNcdGRVqqm4&-n9QCtReI%HrB{?2n=8-W$RRbiTd+GLkt%W*?6wM-VEpK5c}GU#dj zhvd33n=P}cvtNJR&!xXZFu;WPBc}|JRHwMQ}Uk5RB-B7!KrgLE4>#y-27!12SEbROgYeoMV z@xfEX9mA%~nZxMbv3a)sLlqpj^Ji`tg_~*NM!GjeqMA58ll&A6v;q{G;c10Ye|-O_ zsC3~v`@PE`T}n{cug7${3~l>2<%!4fNfnbdrZNs>sTbq)q>NWUmMg`UiD6vLadg64 zktC)_tJ-$BKEl4?J*-3>dD z)=O-(_gwgKuU;WdU}>eQpX>#^ z6T05CfC9=VwsSA}NI5`G=P+OiA5TN72l_>6KRk6zW<|1&J`% zS$IkUD@!OU*}s7_drBv<*K`yVsH*zxOt%?{B&3%!m2nz73K;8&Yq#tn?J}4J_cnEc zSTEzNqnL*)+@&mNb?`NtmVtQ}*N_8yi4c*22Qt(V)cz@NxX}^1de_z@d4d6go~cXU^+C~!nUEtgo2Lroy-FjAFcJ_RJX{SGn~{H zM&V}Wvk)wwTKv!bb&YYQyA&n! zVGNUK^kP0w$1I9r^th`+2PG093v+|^jcef5+~XB)zyv=CM2CGL;cy+}{QS53Y_pn0 zdn1Tq#JGAVe?C>!eX(W8V3-khQv&UZDdlzzpS6m?cPc9K3U|xnZgf(6IjKgthMCqW zmCDBxKDQWbeAf?XF4p+n9vjCTugKZwn0X9iAbDpm!$eK;WUK}m)fFz&gx2;?%%y2a z35C8zx-z#Phhf1QA{OqVFmKjFlZaXw-!}Qhw0}r7ir4(8?mkN>)VJwJ2v2uZsh3~y7%PBzRn(b4J9>a`?X4b?7aLY*okOzf zL>8>S&=v%V^Z?)9->D*n`|I0@d8dHW6#BH^<_}Khy5)4A5+Jv*lD!NpjCze~C}!^@ zJDYlqmuCRF)?Y>Nk;uR5|F4GM^ENCS4NtW?rr8ffByefcuh0$~Zsa(npLO3PbBhkzXs{qkK1$uges!_G(a&Q&!3e(D z&8C-D#&|GE$wAr7qu3gCw*HO9_Q*+oeb3IjyRCvF9+nPOqS< zU%a*@+`k(s{X$7c+fIA*>$^+Evd*I^;ptxRHm0GX$K-JBD<{A6tY3cN#~A@iNO0B3?&_04`?xe6iR0Ibnh zg`0$$PznVar-QZ!M2FcgXLin6h|8H(!iz`(lBT9q$F(P!MIMCx;PkLZ>%Le?^qCIb z;u$HGE)iE;cZ8k*JpWyHp!ezriGBNddw$FPTRua2SsDcTMPxquMPwF&C>D`VpqP+Y z_Yhqu?CNW|hxB$i#X8m~kD4DC%78e#F~>k zv1bQHGrYV}pT54jQGII$Ahl8Vt)}Sc8_2rH_*-9*$?Esk%#j-Wp%!fPMq_b@1-~SP zpKaawB(|EOH7=hPDG#hb**Hy(FPQl=>AQ^8+!7rE_GdANG3Iip!y(>6ciW9{V4K{g z;==7n8u#I;qR@uXLi}~SGAvkX*9vs3`%(8zJE280i6SM82$nq z{d^8T^U(Rg0Yp3CC;L?x%=8zPnv>T_@4DOiY0a+6m~eR|R7+G$PiIZ}9M#^^vP`g} zZqjbpX3V`=ZuCL2oQm)J7ME@s4ZmDeOhY%gFj!MLT%b$Awbhf1=g~{zM3U=R6?yzi zKeV_f44+uTiUZ}5-t8`gtF+H8vX>D<)n+t`f}P0Sne=@zzu#Xg%b3kho3b(JusWyE zOnD^l~kqzmmL94;>xK2QR^9{+guxoTj{M1KmkKfP=6eIf6V22_ma}I*xyKb^}*sAMNd;J1-(W!Ds=rL@F zRg<~7ur@p*F8BESrkelrX|IBu0R4qBK0+}0XvlNdRCF^;H&C}bNgbL7 zkAA7CDgRiN+7}a%5Z-8ht|Cjao4)Hh318L^*OzLa50Glkia21oIb4t{3Slr!dG*#+ zkTTy#d;PL*I z)QXugr|ty(9_MQxXm53leY!p`>FA4gw|AEDz-{vENZs?(6DU>mWdXP*giJb5morPA zf4?06OqO+?tD{n7>oTeD^V!PN_D`vyjS8+AM?NUPpM3o&Qi$+?`n3{Re_o(W zTv_BOPCGnl^aS5KatXakZq3DSh$2?XN09^TsnxgRIQHVg+P3P?6Ro&0-p2jRMT~@v z4VxaW*oN7uu!2H8#X>d#Ihb2gE*$XP&@tF32sb+r`Y+sEV$Q%cfACSb;G}f_3;=nR z4)O}VUIa@?PWkSj=JyzM=`(Nwrv<=lc#i|p!X1xyy8m5mocMO?pZj}C`9I|nQ3D?^ zZ}*h>zmZtH**54>8wg1Ju>SHalM1vXPW%3Eahe`Ug|SG%RZ#-% zIt&3M4CLAW-~|YD0^Ip9Kw85x*&`W5mn)Ms`@t!;M*nW~ZDzCoh5|JH?_^K_daV|i z?)2=IEmQJ6-{bLb8&JtFIjLVBo|Zi5ahf zm~E8-6WbN{7<09}|NVlYXNSvv=!)&s*U`y#F14PR~Vn*0;eJbuSLB6)Uv&6RE6YsIt8&94XY4pruqEmv&cSVp!M{~^L4O7!& z0ut9Hw)yC6+ncsb;28U3$TwT|KS3;ZnbXfhh%^jRVkpqiM#45l-YT?A#1-L&shGe%`C=gp)1f3Xz9i4^;OgT7RKxQ! z|8uqC?{?^4QVBf;UdjJoM@+vpeGkoaSW{eyzQ@P{o}cA@KI?+wjuAfq2nLSbWnjRt z;%-t!Z()~!p$4XqtOH5Kv%Ya&{m=MqZKJ4|<~zf!-~75D+R>x__4G4jha55>dm*5%S0~bj}c&|MGN-f9=DaGq+Dq1*yotT0Qrs!pKJL&op9y(xTm13c0ll#p@~NT60%3U!NW0VlYK8*^jm}YmBB<} ze=;rydEr82S)1@ToH-kiUsv36gFiDRTC5r}tHOFN`aaP9SH1s9?hUR5NC$o;J zT7s)}20^-SL;(IF<|98lB6Vg^XR588U7u!NQ>wR2gMP<)pV+p*V?|Sny)oOHEb3BR z0grJt1%hIRwOJ~ahwXv$ROn?ksIFEhwBWE4swn8NTItt*(wdTwZX4G~%?4L(X3%tI z^=y((j=Yiu7={_JW7xpRXzFpZ z(bW}UAat{R2he6B7HaYzmbLI&0zy9+C^- z9{Gk42COU)oyH^N$(T;7#)XfsUr#shh`xasJ%7^Vs*cN9*dXS3kz&Uqzg_rv8xG3j z#bNS`5^9|X_-!o0#5n9gXSo6Ou-S7jLD#w7m(Qx8zRbkm*hVfnHgvwc%Jl71K7e%J zHeh=lnwaWG5qxu}JgZ600mPF04S1HP<=Th;5Gns(a!#WD1$Y)KF{)tFZL>YGXK@5JxV*=_31MWS`ly>tH$=y_oU|K%J z;Ate47582_BIDa?gjch2YJ-S%I_Mb)^cNQJ*N7~Mlt>Fo???+B1cXq7KmbDk>4DIzfMvgf``P>1=ljlkzH|OK-}?QM%uHt1TC--& ztTp$2U)L3Z{#@O1jP?v97w@5cg(r6K$otg?&8l$GSI&GB7)>>cNfz=-AQIT*(^RDn z*=sbP6e>9r+kaO^ZZyB8NQV44bM+$LlHriCeBbgD-HBCf>(7@?O{_b=9s7F^&@*jv zpyZ-_&-BwSC~XjJUGGGYT z2MYr^`A^M^aL5Q`!p%pqIgyy0l+6h;tF2PQJ}yhIRks6j!^rfW>Dz2Q2X0~@QvvP4 z%)uX^M=Gp8un`~i7Vim2x%=L0yA5mL%{Wiy^xcm=Lt-^TP!r8uV4&lS(xJ(D4uXLX z6DmyNk1i92!oPo~xR|JYAsEH7FG08a`+q^C1uD*VvrVHd+Au)HYaprGYQ5r04Ih=$1aobjywfQP)zF^Ad@4HW(2qJx(eT1?}NvX+F zne$-7{W9mjmi1@atNnPSD{%9{w)F0W5Z|8nAmov`=a^v9uVee~Z!7cQN>`KehZJgd zo}DrU3>g!J2gJx#oqO8>|MCU9-Y3$Ae__lbeo+BrU zOxz!tAnSzb5eFN%hw-qXEPWSYLBZh)@Sd~7k}jk6az;f}8~f&hai#R`MhbeOTbvh9 zncgRkd8^k8;5+8~6%~=|qSe)0vr)7A+l~X>6TJ8;VVI5!sUb9xB5T@NE69T5XPrSO z7c_dC507LwnuiN*&)JQgfi1mLRdCWW_uG&v4nU_?*IM4m@z6%4^KJ9Q)^@!b@RE-6 zi#L3t0jD*WrrrD#L2>L{?A4G?z*Tki^$EaUG;gq(oCHi_cnu=VqZJJWZWx{18mj5J-Y=o*0mRG=eu6?bm`xHAF0O& zLcRsIfkI|989vg3vhf{x;J)dr5_fzTJnD<%XUpl0Mm!{*-i}bh%2I7O1C53d@VP~A zPto>mDX3nigk1G>FW!s6wea*N3Xs(tIx*D~^s{*`vyJ?*Hq5zODyYqQTMH2yUDcu} zy>8c}IRxSF&p)RhK}wU~^z9fgcPN>0wrF5<^B@NXd?q*zA%ZZ=#@Yn=F@OfxWLSY? z>S1n{VV%@u1%LbtlHQO*=d{OcXL8QWKHD`oz4gV&ehElVi1CSs_*19a-P9STq8M`$ z09j)FCwvbdGG2-Uw1^pmtNO!b>mSqU-N(oR5BCvc=3DGq%9gM14;PgP=%GjF9E_Ya zt&;eUSz!$)M`p3CQ=`w_v=gGeMLF}61K#yj_4kGw3+*Gl)YkIJm&;w4b%fu+2CBLh z9dv!|Gr`l+do}ZRIa=ad_ebj9Z%e+*XUr?YW45JY&|1~vwYydMBU#zys67kRSJs@M zUBz-nIa=4fLW@{${?ij9Lpl($V?pn%li9?%le2R^hr=G<#!+7~aX9m}q$bW8PkXu8 zMrUlH)dtNORzM@YA2s;SWNsTH*_atT%7P8=?upU4+ifQoheud#6P|~#2rXUTTP2)i zPFArD#Os7a9wtNv$agz9Yi{-@x%Zfuaqg273kHL8zAk`+;GD+vVscaFD{IsBH&GXL zykgjWAtTO%0`;8SG9RjWpcZvw(<@0Oa)Jx@hoVtUQtqiNQghxLlqR_-Dkbj%>ziCs zT;Z`3uTMx9RfzO3zut?!pRvhjkK-$3KK1*hYui9@-HB%)&M~r8R$WEb)GGVorZSfJujogyaVj6zz+bL(tPE^cHN9E|%Q8sCH*9z%cR1LuY{INSjDWT6D4meL_ZQjN z(dvIB96Q?bHwM7tZ$0RcL(s9KzmC#xf|mhy3Z(S!8hn*|cRugV4s0vpFgFt3_!wW| zMxN|y-cfmRWYh(yBX_lGTGX>?gcrJqXln+$?kmP{&VSf(`KuCS)l0w=f_)Va=sJ(4 zy8n#v3XjqJAI5m0>?>t*D|bzE6>G*ZnjR($TyKGw_G6lctaAv~$L)jP*|~ZfqFYJ~ z<=A|rw~m%S%xg=9c}QLx@97+8GMN4EZRYL!ezllT?&Qij`c) zwp#3KVAbWc2r3HL#eG=n%s<$QQVky4 z89UJATu>;yu+V;y^$+wXKzQ%}8pQt_9;7|`DZlbJ3i6XP_V-NG12d;O#_~m37ZHHb zVVc|3y2}N)*A;GF94N=_)fy`^_jPLvZsygs4U}05Zzm#Gtonc1Zv?oTL2aPYc%WAPHH4%9XT|P6aVG5 z)JPd?PF-vw?L_+9{>Rw%oO6-~Tpt?1mn50gm(eOkAl{=a|DBG<{7&m$^o;$C8lWw& z#Tdr@iCN>;uD-P^;uf7$ch1veV4wT)`|^5@DlHjvzjR~Q{Jate_6b_8r+VVBVeI!J zUcM$i>#i_Kmq|&vwjYkO!*TueJ>U(yQnE@4*tAM1co1Qg-Pcsqh&2R)pnLS`_zx*Y zaGiVg;B~w7Bx;g%^bh~?`S{(X6$f6y`}LL8B~tx*8;Og8Z4r>wn~xzY%@I$FJeR2E zT3#;pzi(sJ{oi-nsX*AL-I^wD1q0IOvYNkK7uT3IUNHmVRh^(yXUuFI(3az2cT?_D z*)%+BIg7~!iL8YuuF=!%>H@ENb%;o5iPE!u#r^@hvI|7Y>YAfU6eGlr;g$b&6Cb+; zI&r=W3D=cEKoi2Qe=PiSR0Lh~u!$udcM!&JU}exU>ta5_3hZL);_i}p<@%Ts{QJh{ z*z*x>twh!n(hb%)V;^1RFR@Y;GnRh$=d)(3;;J@h${Uyv<<=BW?XydMFi$z9o| zuQOw$cWo+xbhCmpy_B31{Ux8=0&VeDaEt_J6eTeMtfg1Z_+yhFEEp1h$kD2W+7|2C zNi=+p#)>cp=QcfvgDbEOp+K)sy4|?1tOe-H!aBHwF=T#`l}+kcoPk?_ba)BBO9Io*tvYrSBMYMiaB!6k|swuf)S!8b*F-DqARm~;_SS_WN zYHKU*zHP1z*h!NiJ+7X(;(!UR_(<#;He4(2mQHckY;szQA}#AhPL1eJ?WMWn@U?d+ zCHdMnWd{i0KR~6+xuRZwfVkt*Po!Af&F_-k?s~Z0D6OC^uMRFIv#A|4+3EOForl{i zmkD9|T?ApXaaU1(G{nfyQ~4U#`+M7-Sg3ApbnS!jbR6T755acXAaf=a;6dE468vWA z;%Avt7ib86urSZREViS>1U3gdCl{-C_MB4{?MkzsM{-9Ve!K^YD7=R0M7}E6i!F-) zKEh`&y!f3%oky1d3BLyFf8EUof}{m*=R0QH+jWa!(6t39a`uyk2nvY6(Sg`>>41(bBKRhk!}3bRt2;d;U;M0pK2JbU88@QK z!#-k9|IA(-_SqE`O-;2FL)&jyj%F}PY(2er__`!M(OQ1gfFJ{0xS#c};vB@Yv7ctw z1vh0Fjqs+A)%Cvc3kz)omV>b^c9c@6{Jb2RaXwA(Onhdle7TO_&F~+Z(j>wCue)Qv zGUNhkY~l6?h|pZ}$zA=Rebdjv&VVoX(m;NEiQoPjR~E?l25|oWg-;Za0SGKVc^Fk~I&c!@zRHGsN-0a+wNJ^*E>p9N-#~u?=STRK7WNNa8MpsrAOwW&`&Ltad;l5WEqOfNsye8k^l%gNn`*(M-Ruq+UvXByLnz@`~%9HE-}bK!tiJ=smtE$0;~(H0bD*iMrqRGF6-lA z^r8r2*)=ur*{=9*jDP8_C=>KP%DhM=-EgVBVy`0Cg&sq=cya=a_oN$&rxt78JmG*SMh#H8fZJmRP1`@?RjzsDyG z9|N^`I9qqcPpE-uZGRnArP++sbXPcC=ruc(-dK0$jKP}g7%DK_wBRcYDn;pcZ^7Tu z(#(mouyz~1Ysa27%QrMP#Mf_SDzb)JOz{`SwJH134xSpc)0q{|l!8IE2naZh_8ROF%{>up+`iJ-2?{ z5XVABoPx9Bd=c32ta&uuytA3N0d6e)@^ErT(1>(?No8UD7zMnEJr4w4!v&|H6B6Pw zV6~#E&gr=x9~Yx4*Wn0vJ$tI^T``xqT{sT0?HUx}Kk|=B|`A_DdDtV6exfNw`l-k!P2t z>KSo?iW&P$RB5iAj-1KwMf`&3Q%Ppt_)*TetROQnJ&gU$cPev!iu0@j$Wh0<87NvG z!Ipp6S&;1BO*k1GIm}}=caQWnh{HP#E63H;#FJEuPVp9)my7lg zfA2i~)njaY#}F4@eC9oQx8up#+PGi(Mvp{qUF?(|i!|ev^*Zdsh+B)F`Z}a4@4|~4Qe|D1q z$hhQvvNx0!NiD|&x;e8o@Vy>Vn1p=iTl=2sI}4UtHZ!9ri$(*Shp3*j_!q+>^*LGD z=T2#+DztXNC)-z}VAA6Oqo9{yKi5E8%pG&?*yxQRV)kXj8r~j6JEP42A z%YOw*2X+X2&dKZ-XTyB(+ARnSyz_*mQQRG&h^Kl+{5NA4LaNLdQX z4~({^U{m%>ULPY*9W|F9&&#G|#^vt*Uw4dVF;7KhkkDwfVtlFzfCKm@^6HH50C+{>D*`0h*mw(-SdOSHV#! zN51W!SLmGFn=N?`dwU}H?kYgtt8FGr{Etd|;p1w)B>ZC_HCowJgp zZ$ah0(Bo$>?>#s}L-(#i8A9p*w5>mU(zlyvH!bP zsO2iX5-&i0ueo;GZr3>S#;UWN+lm>sSdfkC;VQl>m81QoHKqFLlHscZccpKRKV$QM z9@`=K;5O!z?@>R1IU1^85~#4InI>wCJ1 z>pHXL>4y2vJF3~9R|B*#&k6qy6vjQIIS`?*{mG<&dEW+| zwH0?`RKq**2?8dRNV|I2&bs*c$Rxx$n%BBWwrSK9N2{J)>|%Za@GM>$O|mjh$om90 zbE%}BJL{pL-~*YpsTew4+|&oLS?is}F^Qrfeb!!j7yuSNf|^0~w5Z~uo_3WxG_1;p zdSDU?os@hlhM^4fk0kD-Ux&>G6LKNjwO=0d9lw9n?M``9&b+I0*JKS;#0lhl4Jv_c zAh5ChUFmeovzyx`DOK_CkZ~8yjsD*pNQR>P{K_Kni7XaYevLF^AK%$(?|TLFj)|nb z(vgj)yiwbJg4F@jn_qMsjLG+TuLRlmkmHR{!DPYEhU#>6Yx1b8cG6~Rw*uKVnXOps zGc_(yZ%c)3xl4X)zL~40?)Q6U3|Dk9E|AgN3rTc*fF17HQ58X;&Ep?IpbU0P=H8g| zk^g!Zx{JFYKGm()+You(z~e@#k9+E%o^$ofcf83p9Za4A<8k8^16WC4pb(?1X~uF& z|HsU78KbJSLQdne@$uhCnJj~mGl$smOqTb_f&D7w-o(93U6g(Vfm8OrM_FYpz2+lz z(F^KT`RX^!+_P6X=Msxj3XIu>s9?twfzR3a&jwi(EhzWN!H-UAVuTEMotd#knCs;V zak*lJwL)i=*zUQ_&hA5D12Y<5a*sVqK8#My4g!vKaLMP3^YzvFxGGLD-RTC1s|%_t zg!x~@4Gpu3*Y?(K%9ZGuJ_d`Iw@!a5fa{>82z>a}esM5W&y2-%cB-dKX+=N3ubMB_ zZ2E;0To;3JR20o`jj~wb=l_~$bTi#)*CQvD4AzoPnaXUho{MrIiHaM}bZQo5D_Zku z=8o-vx7M;U(<5)LI4ZA&;_GtHj0>g}E8yt5QAq)h3#-pX^oQzJxi>oNKClmfRdkdn z(CiZiO2`Nb#j1>-8}dFt4%RB%f5yzWXg|^4k<)irwj#NJU=Btvj28FJk-x~X)#^f; z>>o|d=2jv(Hihyb+eF6cF}Y?3@xU6aSXR^Q0Gg#A)_7fco|S=RICXWZbEn%|xK=iW zaH;Q+BKfiqB*(}jVmhi!4qhMjR3F#3IWfeaX~S2+E>M?aHJ8#8?6JjLH`ILviO?gu znctmT?563+#Ex3*3biqb($5v-h)9SED`3N=Is^snPs`dNo&CEF)%+=q*;Ifw4h29H zw$cTlvZVTZd@n?@WGzp~8W<|DJjUs!{H{`DqBZmpbMbzhxUgITE=0F*CMpo7?$B9g z06%v1#l5$``XiIGX^Hu`ZTY-hXw)ocU%Om8KDuObt(1)tmY$T#yl!=9_l+}f)W02t zTa~ZL73u;w*rR1L3BpCCyxL5)P9{&lOnOm|s%HNHEiKd#3=FCvEJLjZ7E1@^kN_;FmzpBGgW zzTgFC*p1)`R+nP%(f614qUnu5zlpH(`C@=^F`Xhq$&Pv=%q^h<#Zyy6!tRn6xSLsv zES?J5kNc~{aE*mSr^6R;1gkrj;i{!4x6G~H=#XFa)RGn08V{#aOEWUP0-v!YqN6e@`1Juu6%_n%m35 zhDYtbDfr6q#ZPONeEz_omtWmFAGhe9(Dew=v$Dt?@?p^mjJ5&oRjWv_r#wUgN-KQp{2Vw`$OBeOO-QuA!*#s z`FwTcv_u}MQ1qpt`B1P0uAROE-Zn2OV7pkY>!nW;>X1|EzTnqGBNEig_7JuW`_lE! z2-;H*xuxQ)lMLx1o7>3Q?D)pHea}~8&KPa7Q`qDo*XtK0XQbwWud!)dFj0%5R91r6 zZ~a*X;QMr}(qzg8hNbB|X=FBilefBZ371(>T9BR8t$1%pKQ}a|IIi1%K^if0m_hp5 zI4MDCbnErId87WDUt?NfV#@CpxIT%@rV_=mLf@xPd7pF-=x7($efRm=crvfI+`X>*PgV9OcM*Nxl$EB&_n)7b9-7SVEJx1mhkTC3&mZ4y#@pl z7MT`CwkwRD{J{A$oUWxKU#p>EK`ag1fohM9XP!x(abf7rN&WHT{mX>PrgOX8?r;YO z_b3Fu8!9aK(uz}mX7K18AIXl2)R{ABSrVb}Uz)CuMEPDq`~jT$sFU zGcLrXp%TGPfn7Uq1nG#A5B6MeNhQMmXOS^Rc3nmeC0}b(_MT^%O5x{yIC-Av4E|FhuLd6ifJS zDvoC-`wPgM*DEWl#y8#HC90fC8afRw(z4*X@Yz4mgd9^H(-+?(dEeBYUPM;8-FY=FO4Bb-k<@`cO?n~2vMJUolqpeX8V0Q+6OYt=gtz;hxZQ#U!@ zgJv9{N~>L|V0Yy{eOt<`o{|!Q!1ykN?uu(==*4v?&Z5CAUOk+z#Q9rq^0wIYcvs~0 zwHG1X9@ocVnVk^Vwl24MI9|85H4TkxY$!qn(#0d@&%|GPM|6ExS0aS~B!hUxMKNRb zxPS(ag#d`}^2BQKle$-!36@&q!&ln!ziTlnHeWIs(l#~SF%jzO(s=1qH!s0H-C0xt zic9^bQ7*)np%)+gNL<)gys@v)VQt!~9jl-@a&L%Gzf34S)nskzIdO@3M<= zDEpGb)$X6XXUQZyFE(8&3sVxYFEgp`c8p6+eQ&JmCEjWX9YH8uQCBF~!o$V(A#M`M z(I9_5@6O}r2>pMKGFIZEnuZLO-Gf*^~R z^ZiF`EqCY%oXJ8vRd5?KGt&)3lIIX{I5?eE@IyMM>RAQxS5efQOqSmz6ty{xdlyD% zf@d-JgLFSS^nP#0p!MuWg^X?0TUOc8t1@X>2k*pQNy^FeEmb@#vyYWU_ zIuhj?Cfm+Eco54njtaRva4i5n*fl*@CGr68m6vS_N8il9{W)@kFcOf-Bzgj24m!^C z3Sd(mFp7LVplh(v$HH^T?uFIH_mE)A7=5r?Y`*69-AF^VSh{!vO+Mx$jw$fGU>PYC+tgt32U)=G{uiT(ay>=mPDmWRSIz+%TFpTc*)%Oi9SUi1@Zb8GsT zWl_U=UIj(6$b9W1>*qJ>E@D1!j1d{8ccjbD>*<$OUGCvN8wUo0UJ#A?h@kdqT7%sy zSM2NFTsorMbiQ-)7A>+STM(VgBN!?4$gRL(Ux`BbLqem zdKg8aA>}HLe9_fXmf7=(tbBIiP|JLMPnT|JlMb2i-1j487|s0%q9rpQ=g$u|sBaH1 zVdPZeNb@(j6KFdWfaDfU!oToaH^@m#q1bcBoNI9UZY?q4RLGxTXVPcU%z`Llxo(n> z2cpZ{jjiO&p*BBbl_C6 zyz_c(C*e+(ZjAVDb{m(`hvP5ve%-l%x#U?$=j(qhAN~9feI&=Wj%}TAQ?=!&X*E!! zoL1D#8bmboeUWB5m%FoNr?!>x`S)&GVLZ}ex5H$if!GLHD%bK_O69Pq%(fW4Sj1}I zzFvGD9_*q(fyCL*Bl;pM`cX&#+%Fk( zeIsgCoco=+_JvZL5k*4(>8w*jk)fUI!|;49tqa*Lkel%DXbaKu%GYF-fpYQU(opu5 z=M@nbySq0Pj;1keSB`8lp(+)?EGFx}GmGhG6u;!)Oe!_%amyO7k$JmmaFcB>r&_Zh zK36RO$(9>+qin^e>O1NXz1NJssox7Y4U}*0dHnRgIZf>Uk@^QH?d8xgOfI4Q^19>x z?C$k~Oqgx~C-3EuEy&UnroF(dxoJpsx^u9<`RY^7Cs`f3*b2U%_qYB4{dIVnKR0n` z?+Gxoj7t1&Ckc7JG)QIJh{qh1E06tD{GknVMcRAqsrTKrnZFzo{*d~21z3lMjDBwr zCeYxOiDsI9IA@iY?{Yqq>0nytL`Usha&;^U!u1_om>1$csC_%iN+ZCMz>~l5v@I?D z8{n8!=Gmay{UkmmwjTQD=aeTqb{3Y?PZLGgg>!~Rvod5!$-#aCTPX9-N>^6XDuTyz zwtjd6iU&t+fyrfusB3;t0`B9#GEZ3o9xj&{gw zES-%ZZ3Asi{MF_$`yLg5w%<+aqaYPEe*QO=qp?h#f?!D2vyJZ%lza7qJ_!*2V{I)S*5@7FsW6`UbaG@ z_0R__RMeHdZS(77-z<9?EEgEkE~0cq&z)xd4kmx8NZ)>s!=?XfPp2Zg4_nQDT}>RO zL@M8)F@dYu?c%S(eqBVJzl#Ehvci;=kNOYm5Gg8C5}CX8h;BfJ~A%7P}>dO{DEr!Y4XNCHGMKCw_*WYm25&dpR>0f!-Wvks7 z8ah)5yUoBLM!$XyXsJIdFaPxA7(G!hUN8yLY#w`*&C{c7By*ijogrS{G{;%zaA7&I zw9{c^**lt)YMx{s)-ctSI#ka zHT##e{Ub$k&bSo0x|qy-zc8e`FyV?5ZujBL7Ebiq?fZINBgi-xncczp7bzTGB zwD(X+Np3M}VnNpzXH(zX+C?too2hA+%@{J~5%Cyjq!)*R{-0UA=Li!i`Mh%nIPil5 zXF)yhr*5dzdncqF-cD$Frh8or#N>Jlcr^cS0v!N*n!ly7VO|*E%0;V|+II!&ig7&y zti=CJXR~t)v@~19$H%60mul41yMzcYBWIF6G1Mjd_3LQ4`$r5z6=BeqN(Dpu#>I3+ zyMZtyTc0xqcJq|ga(TX%S68><*QYDmY+rd1TeQ(z(glm36ZNCgX+24Kw*W8&&DxU_ znV!554dVP(lw*=Sp%qhw6J2mLk1ap*0As(OD4oXLfJCGP1hNV^4%>}8oP889QxfC3 z9fEE79vyCreRyO7V>|a6;GqOKblL9`V)^d=B7;7ztdezG;K=~p=+I3^UR*mz)s_a6B}Qr;?xj2hH3DV> z*2@dtauVfpcD*A>n`1v`X6G8&CicQIb9AF@Qse!Q6$)>QO|s2C*UClZpv%k4r8bq^ zi#O521B!OT`9%w&Y!XFf{1H>xGXnuxR5Y7LI+szVuCR)n*H@H_a?Y2c@9(X}*0td< zOYOeFNlG!qJy0aQa}G;k!*XICgy%_V;dME< z;Y-uj4j(BP$Jip568+%#L2Qw@MKq$?S2{XDuc-T`H$QQ}$$?Cir zV;aPE8(F5CTIVT+jzGopiz=+lv(#{iYj;w0xtFM1`m8_sb}4VqC^<9Y*>yc4Ng7#CmbDk8Qd_fJXc# zpo$w9$t%A^)-Ff^BotQy;3h3nBx6xS`znEQ0J#NlTxZIVyn`M60v>L%w$COFn+?+V z4HJk)a2}+qGhrbVZ8)T2DA#Oi0&h!H6x;?|WY(@K8zyRw6zb_x3957@XqrXbKo#5i z^|1u^g-QR5^ctJPse0|Agx9g+lwyZ`Pto{M9|`*1pc7tlatgJ@xIz1zrrF*hnzfFd zAgy6?7TbO&Mn=o&#VBpt)}Rok*%WvR8&~F7x#g8@*RW^7&91!cKT$w-eem@O8_h2z zqyCC!yIo^UOcGg`u-R)gzh+8ZkWEb!)kWlFq|9vKxUU?4kQO$2>>_?cq3JKK+Y)`l^O#~}E zeXt=A2fIb|1IFloch*0b&iz9ef_f@l&Dp%HU>NG^gDr>ogbu?g+$K_yZ1wmwIG8gg3_GC%HY;f{Nnu)k02|GdIZPB36tGo?jn6 zGf>p>ey|DlRcJ{JS@v|S7_KT><@l~}_!$IPNc6g=oxF*Fq=mG@7*aTgYN7bMgy@{- z1x$CMuz49?1R5I1a|%K9RQFczwL;AeB-a_J`5P%V2&=h$>nYwz8FhL7(c_fC2hizf zbO826mjNL5eRT_T33MzCH$YdFI$+@7*+15axxmN#NJ+liTmKbVq9LhPR8KyiAfw;8KSj+h zt(8PWcIqOzE=J+_dqlzAc2%C494T1dBmg6;IPZ((EZ;ThxkBkNCUczu$RpNJ>$VlI zcEvO7_Y&&Gy-Qa8SFIsBI`l=>qVE{<1YgZ(w=$$$^dWhye6hk;mf^wfYT0KNa9v7S zlV&l$?HXlFL|b`u82S0J#9K=|ULVa{63kYh&>s%y7fafE9I_Vdn{l=rpr(xL96vj= zo9QQ;&8DJYZpd97m!i11Xfqprfyu?$U{t81MGiASm?i8URg0Oacg67~bb+s9qU^ZJ zHA|wV>eZC8C+#WI(=-0;r<^FhR1^?gNz*}27S*Li3yaAn-yqqGHX@75_hUPYNRN?t z#~@oDJLk=$KR^i4}cT|d{&nNU@cjjeJ&J~GL9{Ot&3-ao(=7km!)~vN??ub{=-lG zu&V&+XG$+ceNUy#mQ5j&UA`m$o8*BLZ=dJ75Rj;w!jW+GHzAKkyyK0TZG?Mz*u}RU z%83OIdS(ct;TiZ|LV9C;jDY&*nrfDPuge>?b}eDnfj2Py&$oX#dVP*|?s7q}Una1ShZ@_OJ5lt}Zrt*JQEz@!l)&x&-v-0`8vz@ZIDqnW{8X!5e6aTv?@i*`=AY+BxY z;~NRDb+B(zFJ;Us=Bw??^==)^6C^C5(G+&ev)$**CA!pVvbOw6l%BR560ZdDiZTY6 zZTVN(FJ&(jWS$gRpQ_q0M(g_JC4C~b+*T~-~>i{dP`Y5%|;Lm{I zBhS@eG>dotu*RBlM_*Q634CKE?U$_(XoX=5%J2b3exw z{dg-c20K!_K2}CU3TDy`Vnty>R41#7{?iXRuWirG?%2Cblq=Wm$1X+#)Fk}~-b$wP z2U~vbYlXviw5Bg3Bi*$i918wM2d$E!5GHZMs?t_-Q4cgPBF=BVcdr(Gz9}%(BCL&T zTx#pi0Jp~kLB$RIN%0P~tD~|QlBcGc9dzy|N%hs*@nF@!R8?R{V5M%TPmB+v^kMp3 zx}dCVUo(7~i(aTm;_izsUYrFq{UPWV`u-n!@c&r+mo$80w5RK+W%dL6!8l+TOw;&l zM)Qhy6yz=x4hZTjL3oo*llmtJ4z%9!w81YDmIZYza=dyXGjrQua3EQjzl+aw+69Bisn@eh?LY-h>9hCUh|tBPB1cUn5d}3b!-$7PW=Hom~j-9a=Mx^xiVjJ zuYJiyKX|sFLin(2I$tt29?43P=|5EnFjYRe>XSO^n4jYwRb9;jgpLGhb|{B z(C}t9Da4TjS$)%4aa#2ET4-+n1V@nQZcRjzkOcruz0@+zLL4~LXYi-zowylHL zd}^1hMV&*i(2*!pE#M`|M>oF|(3|lfK-x1tRRmJGiiLhwzdYcq)EcY5ZR+DxTw4Dn z$s@8~uQ92-TLPhc+1;^HFog3}dvZpPx$ zIwprmnvXha)GM0{kK->I-Ij?F7RyvwGYvPOB@T102d2lu%-0gkz4#|7;${8()UOv7 z@g1MFl?u6eO0q+0+|55g+L`}w z)7Aj$edzT7UKuE$DN1Cm9&9^!hIodEnj|-mgHC!fieeD#Z^Y{E=PVQk_2iA$y*Z8; zZ({r6PE(M`0ePshjTzm(;2w|BL?-0w8t#gUB**gzc+hUjSMT%ef$OjQ#Oi$c(bJ_W z+^7Qkb@*={mZ1vcbLBs{miBD<8YlQ-fBB)9r{up zCKnHtb!xeK;Wh{CQ4!{rI#WK0iWKkaL3gR1oDM|t)BEGCCC+{-T8`SUT9(k;RIhFt zvWz^QrZu4yWs~jhdIc-%%mBD|H6eh>Ma;bD88H7luQm6Z&})FC`WU?AHCW5fK)aSBRSHhZtOJfDzzs7vL zy}LbRm7g2ASU}KePUNfM!yXqlZjv(6(xc3Lksg>hEqvQqzD8jC5(?)ezA`VD69aqG zuI%KyNtCvbyUpw#r59gYE?Vz>&Gzye;zbGKWLH);W{L0m`*0aYJs|7`QzE?(8?_UQ z<#eC(v_XmC`D5EA%(d6n%qpg@_ZZ@}SXx?j7QGd7YLdE}ILo8Auy|kcAg_&0BV52z zkgqadV@tB$uirp{f*8XD=n>bC5jGVLR~V1xPTMz1ehf@R_Y}xRmpxHDNGi*RkN-ek zZsG6Tq*YBg7p&UkSHbdH9`QA(+LgLC9Ax$mD@&GmLGK8|=GEaQ7Jdp!wtI{9`V-o6 zw`XT4SU?0f&cypIa_W^ zDn6B8&^)@HA{wrBR64FpJfiSzInR zd^W)7u;=m_`B5O+k{cIV(qPmSJp|p_N|~0-M^?bp>{-s-(~=)5Gah9ybdoNsmOt2mx%C6fYQ8Lp?j55* z7w^YYr+JdD(rvQeeRj%wz_5p9l)iLR4Ev#tc2a0)I5)KW;X6($T>`=&YqJCKy$8j9 zJbhp#%KP@pogBr|hLn6q9X7q1mK^jmdBxS>r4aKmcbtV~3W6BnXlRtt^mRD-*F)yq z>udV}=JFrue>zY4^U&fr!|K0D`53gf!?bSbqlYBshKo_-NYKE>ZI9>&?s2*i zaHo;vq26I{gdA5Pm(fU}Gq{b@OeK$2I5!m5$E{pN&}XpO6%Blk`=ZY!^)vVGd~@YH z{RoHi9z;K1MzykKbuFlred%@W%XPVR)!P>tL6}5G6B@5H4DS0g@=4N?mfq{@JM^u&7!d3%(@dU;h zAjs-WTrIWIwqTz-ePy@aLGg#F#gjenDT0oMC)H-Yj>3(U+yP+TZp*xij-H1h^xEh3 zqJWm_N7EJRB7fvf#ZMcE_CzI0Qx;vFeF}%;Kc}L?0I)#^4ox#M1Dd650p@PbeB51Y z9X2D}b-;?ob$xTS@t3fEoDmZsN4G_^Ipk`8gm6TEF5(q8%29gUCgqWaq0p7GSdmt1 z)2cgi(Guq8zX%OdZ47J@O?3qMF79P_=*^IA_#BtyeT|wj2c3T;A0`pC&c|nWi;B&e zXR^>NV!WN1vE z29@SK-Sd8SM6mf+itAl@;@^z7%kfK#={Mf%B`u|l``*k-;g?QoyJbU6@2b*6z?U97&d?Tndt=U}W_IkuZ%TjV`{(aT+v?lUf|iG;CG8+f zWrHv-4eqev@_hh~{tE^7kDX80&|f?DjGa>xqz-VBfPjFQ3?M3N;n9S4T0xYPP89NI z*e$U-m~_Fud98*pAfftxAU^>B8b8A0+>?DD=lUV_u)bJNEIl=5Qm!d_x%!qUeFsV} z5~d1cx&E#}tRlG&O1%2Mu6(Rk?(*dsYefl~bPNQEbCvKjL$x>9tl?tq+JDf);Aw9P z$WO`<-QC=k2EItUNKqlhtgU2P5`jlBjsagHUAI&P>t;OMn3gz|tE``fDlZb=AU=hA z%9nIf7(e-GeA>00jj>1sou}iwbnhCo+OagOv0m?7EOFP{bTc=S2Urx#w(ej;!~{x?k~5Mss0cU=1Clc+Im3`7;}#UjG7KOC3=)RSAUT78 zFa!a~k|oC>=j3a2yU)Jo-21+F-o3XzoU*Gcb#-@D^;-X0f7&E|@Z-gp`-wB`3WJd> zmZ3ReDDO4NJKuZExh(RG8NF;f#tuF1QQ066ObClX_tMYR*?zfJv3cP)y#d-CZRS&` z3_C4cf#u9og{g%?v!aeD)UmZQBRLQ0XX<0<95_1@mzYj%0E>5knhWIN-oyy;su_4m zzs_Fk*_+uG@)3f47HgiDw`oD^aa|Y8WJNfVRhl)&Vst!~xfq|~#RjnV zQGh#v9rrZ|Qs}}>H@l@_!srAak!B|mV58p0wBV}vzC5GRey+tJ-dAHOKUb)Ad$&WJ zvk=>F>B<|H=T>(I5@O-5HN91VJ*IUOCUapyk2Vi!iq<37L#1_lT>+(=gTTV_C^WKBMRa7i5ieI9Pse#n{HDp0g;=XKZ!rX{}ca~ zCr$E^?1&88Yk-gXHoz?qW>(0Q4D5vzCyU?Lvp5hIw(s;Abq1gGXlr)t-mOzwy*z?z z+{d`J`F?&8>;m=anZRdsxVc3~Br&teSlYsysQoo!x%?>Vh%Bm{_FT6kqL3qgj0C_r zbSWJpPT{@@S&0m3Yz-y7(?QH5BG_qJA3r7)^G0EJ3zSO;0c&is+nce#jhg^J|X z`N*O=1%uwQY^FXngPvymCfg~t4XX(z-Sk1qcFXJjoTI#GA=7nBSa)V`d%{|o7JIdx z<3Ni{ww}$xwbgKI!G*X6r{m6>+A`pKxR5}g;GP6i8chLrF@?b1{HjHq#lR)h^M)EH$F_sCika27T~cwGS}e#& z*1k`xSARL)`0+42FYFWc0eD1?$D=9UPZhP3f*{MLJNR&HiqHU-R)jgq} zu0lURxU<* ze_E`cp95W_$D@`9nGu7+h|hiP)1NQZKLaE16>>l3hBAMSGJ#9TY#ZhzPoH*Kw}t8O zBs@e{gF9lBAw9Ide`tnAZ;I4xKDwaUzDcl1SS!!jp}mG7?(Xx_ifpDaW7dGS`6ez$ zwxu^}NBL$c6KF(l}>6!cXPuu zo$ZIGA_bg!;nsN>-T)J*ZYJR>6bz8AZ(bpp4<%8ggSVJ+ZEXo7(n#V3L!Waz46SXO zt|*_E$nFk@AHTaJF>}+Xcri>Mr{7uey}x>fPz!aGE54K$<@COaFeEP`5UvBc`c4l2 zKB$A-gZYlov?isw8CXN-)dO~0c#bmdpy*}KrTy}D{m}@c{?3#bt`;7!6St^IP9GRT zOQZO0FH0eU4@ILQlE4Z0f}cSkWj&NRsK_FTjw&~1mGoklAcLv~!BZTm(`83v%0}zQ zc72L3lvi7(osGFjP7gE^dfzPBb*vg{!Cc%~wFp+3WW|GfcT4K)MLxtm>c%^5uiB^bpB9Qzukpvmo0o990@QD(xUmNQ?P25jW&$nBq0a!Yh48YGUwn=I^rO1k-M9 zEC-^Sa%mju@NxEvZ7`+WQiOTsmRzpy+*Xmm7vhn)c!FV@fLCi%N9~Z{l6&NG^6X4C zM4@_7OvKKU`Iwrlqphu3lVPg&V^yyFaCoF}+yrbvJ8NcbX6N4GcewjRFhHu}Pb+X# z@X*E@OZ6=IOENfZ@8eM-QQwlr7_*pgoZ3e3;%bDcDMWO718ROyez8+(8$Aovp5>~_}sc--*$(Mrs&LQ<5mXLwS|V=;)!V%U&*e=3N|mQlugA)mx;%Av-?sKw(w z33Q$6QUjH+ti{5K3k89L2ep2qP?&})cW+xE7Z<~WpX3*S9Tf%rpYP-E=-w8n^7nH5 zEA2w`yg~Zm8|S5`cm|(-CjEnv^e={~DZiJX2fy1#2Pq(UGH0$=M_~=2Fvlv~SP*eaS(< zIlA=y$UFI|>elX#pcMoDCyX!Njn7IO$S6}V5mmQ3Y|-)1WnPEF_UcbL0lM9l@mbhj zN&S&l=IXaYSmyQUVY;0p-r>~o%&b_ThaaJBa#QHmOtTJv*2vnglLVLD?5>p}9PQND zF)yv;Wklw{Z6IvL0YGS06482a7Y`qg8ruAks^#)(Z?q6i7oV=Rn`BpCUHP{WA7$Rb z0&E7qeb!*MZPHDB`sFVM+Ei2k8x;m7sx^Y?4cD|@P@*ZUvtchdbCEFFmy;AqzTz|I z-Xaw}yye*}lD|&dEp7R=rCBsvA;d?OH=1I&5p#8XR>bqpT+_}J{2^7%oArBJQ%6p1 z_t4C5qE#BA{<`3|E%zh7$;r7wu6B(uns$=|Lc{Zdw-V_3-Y7fDm+wXS!#eV-^u&57 zObmDSxIAc4>x`|qp+kr6V^M6ij_@P>)V|?ZXF1GUI#Q``AZtKZ0^OX|c9a__>ziqmL=$h2vwCDI z6cjd-q5OeBKU)34KiHrCcfY1VAkkeECN=m6C=~yB+X&|tUtANDo7_D6E(a3;ji3Ns z?btX=qkbUr$LGXnc;w6HppWx#)$HYY^gG}S)o<-7F0hM22(T#0E;wa-({w0@&g(Zt zHDFs#d`Zj2K3LV47RUbXpeW0%j%Jv%sh)5N9|1?L0EI& zz=T>0_BE+AAOX5_;tS{0d7XIzkC=)=IW&pbvYF-HCTdKv)j^*zeAedHNupvHjWNpY z<`;2l0?%aU$8`HxY9Kt0I%s=z6l>*ISt;ZIc|ezuF-`kcb2xi-gj2;v1E;`eHOQ=0 zmhm0Iu)Cg&vC>=xW-)v^Ay3}jQh|Of3#zWkw6p6b9TL>5o)K%W8=DFz=8pdM}_B0OWOMa$W*?i*ivRt@EBb~ZWwYft+u=)jbF4gn#(3dL&SA3)S2mz#jrKGs{mT5!DhIE^F_&V}@eN9mxx z<=P|625tzofwvi4syO&^e!j1YEGlvzyULnpDvj{iM)jBv)@v%Na81({{aX9Y_hMzB_I5UkJSGwk@tVGY+h1S%$ZlJBN*V>-4@CNI(4ljzd_D7XcYDTW&_g?X&Rnr7ojZU~8}l9`m*urb!wNCGCnk^tHE^#w$`)2o`>SA*EpyZga0$k-gc?5l)!3 z`m1&no%y7wYEHGPd07=;Y3LGI#aOwLqnfI3S+<8Tk~CJ!Bwqt^Dk2^6W0XxtwVM_7 zy|Ze>M6j&hiMGlv@G#Tn^6ziD_`co}MooMjcm1CKG)%i(`kFDLe@oS^N%pg6ExCEr z$>FYab?PTgo0x$8cK`P*NzZ0aY(JcCCC}6SI`gUTuS_l4?hBT^#Y@*5(@C9gXn*j~ zi$4>`kuTZIpi}kpk#nqH<3|f{$!&cGB}i?YvQd9r)biqv>te}f#`|f`giXg>!ai5H zj={^HCeauFa@e-pEuw&~!VC{iikej4b?@MOtF!YG?*df=Gu@rIIB9`C4^x0Lb91!Z z2>^rvK}jjV_Y9R+ zP03$DXtNi?>T@338EJVN23uHL&@Iz|ziG)U z=a)*#wMUV)k5fYO0M@7&>?Dz8&@3@6sZH8Iw;)4i&kJ;s`&WO?EBQ+cE)2KdQb_t1>W*6R zx3;y&RGOIS0Z~|i@cz5xi|4K2y(yC@Dc+jaSK%J6k3SmS3_tei$ZtA zU0w;yD(YM!ib#KXp8p>KfM>r8{__y;rXnYI`pb9ExfQ!OLlYRQt0o=v$sS*cuDdVkn}JM$(y`f$t;9Zkc@YB<)o0eUbUW@isk4>q?Ox5_?on36rohF zh{&rV%lJQ=;uNTR&@SQc)C1bhKf09d?1_nx$Ql%e2^eN7zXY6Wr7&G0p$9t7E_#rf z)!$`H?L){Ze6>u`454Z&k*zJ~)x|BDdFL^~*JeXDWz{Nx;w2MxeKKCRQCP0UTJVG- zg7t9`%35^~)=XBM`GDOps_dGRr&1-Lf+r4v+d$Y4$X~c*sA|kq8B8CZVjmDAos=ua zs&N9l(bJVMp}yr96grXhw9`J*J1j-SAi7IC^2>Z#sSmnJY9PvoYK~Eau0}uJaZq{e zVn~ukPTY$usOe`M6l^Y;`lT%16!xHj@v=ENITH!VC+?S*S^DW`51FG;1Tu%gENJG} zWZ!Lb#>bf{vdWo?hAATlOEjst?B$Pb@4znh66tnRlks~>QX^N)m19>Jaq_(Q#?o3~))7IM zh*dGJoo?ceAjXiSD@i>x8)^yQ%6jYi_jx5vM?53g5Dtd(4NB9R)4A=nXc_E6+bkE)&yGuJn$ zNb8(y;@)aThL5RxI+FXDS#U_0BumbP7o?FlenAn7nMQbe^*WS?YW7tb5bK(|7OM7P ziJesD@`?gZ2;X>L#jXxc-53W* zD)Od=_@b=V1?n|xtPZi_pAu0WBbV8W=M|2seW6|O#hSBXde=fSV&|3fZ&9k;=c zNV{4I2AD*OJ+aR0FoZuMTV5H-_1eBx1%gq#U6RrjI-%eS7 zq0$Qcg*!-g?CK7-#E#j-c&D*(t^4wO?$*{u$!pXzW-0s7$2S zUUA)Y(LtCu9HzT0y6*L46@_|Omu$HN_=(N!OusPBc5{!Rga+?t*ACAG6jb>>6`{Rl zS?Um7VjJ*fL9C*H^rOUqirnyKNVMB9iHzn^h1A;8;mxJp2XnNBS`B-+Z}1rnwV4@| zLh2ONoxon8q0H&feU;YtgMA&(c~l!zo>|#cxvZ79K)%J@t&{%&3UQ^Y(%g~2GuWFs zQU7q-Tkzpml=2O)oNhV^e=Wur|5tSL(~XKBpu3AQuDOfWS^;2L<$Nhs+h#elMbNk3 zH^F5fF9&BhVzoA@C1UP0?Mka)dG>BEvYj4~9$$h)sr7ProD7{No%n(-Mt^vm4!ByC zWCBM>te||~dbnAt&{D6(w=YxW7l)|@dexazH8u5b9uk?27gAE>1 zomidZbzW1wNF1fH0~7`W|vQz`(ZR7F#2;Eb2Hf+nKvu;=Q%jJqXWW9 zDMEFuG3Edd5ceu^Ixk%9^1uXB2()wX?O33JO%7K2c&v}J%q^*N7` z>3PG@^MJd-^}D z_1iiNWI}db$JA8`udDFSrcoC*(S07bKR*%Rbvk!?(?$qk!%99>&95J8GN@V_Y3_^+ zhz{xMuK+SOj~k!oAqa8VbGvcc;?|~I(b>?%DGHws865aJUi8#+3U&Sbx(%fUlFQ=0 zrtgf;1SCZ$-*VN51h6FNIgm<2*mvcf6EU70Pd6Z+ov<6@R?|aL3VT$>>vZ2bs)HJX zUfgAW(nh(i3 zs<@>|n?ih*Jr7{*UN+{Y=_DB6IU7R=EuoO>DhO0G3nrQcuQ#rfZ_SOOYVEab?R&8= zJk^wS4Mdah|KId%W_rIFr+=f z+}{w?E=?3ZP$!o|$C)(S#g?I-xYxVgN(Cxg4id$)Fn5Py)!cg3iHa7I8n&#>nSluW zPCc8Q4$X`texu&+n{Ds+j9xF6gz!hsF>N!}QHI!|T6HNPsTP{Hw*8}SLMnIiXSiMb z88%g-Cw9t)dVr}_%$w8L0$_&7YAn0ONk|c?CZS^NM%Jg5Y~u`9N5sNVkH6Ooow(tA z$w!X(mzVpGGwlqcbZxOTPA)4#VqXp2V;AR3CbZ~CzKbSvpV*e}n3B(1XGWt?mEgIG zjL9)*U2DZso=XV1)_YWjUYD5)Cimh@2Wz zS{Z>*up#nPZE%u|GHvYxplV+xJ>!!ae|f%T`k&CG1HMf|#3W<_xq_M)^Je*VqS`VOqN~GxcZ!+3nWJ~!Ah=nXhhJBzJw3QlAT$h6-oFjCuBfPJkxW)Y zMO~j}?>Ma1%xw3(^yrVibmye^=~g_s_Q8fDsys57rSO}7DI zWjWr-%RfqYW(?h81+<8-9#FK49^SUGWScqX*B~^@0AKhV15jY8X9~-=f}63s+AL6Q zN1Y<6vF-5u?L*4=Q6+AOR~N1NqTyNQ4-nXoE;_UY-DfCYy?{Jq{A?cRd#J12kCJ zKC@DRbR(Il)ef4DFI=dVbcpm+wn=V0xN3t-)fXai{E|>SU>nvk!TaDD`o*Jx_h9}x zlTLc#YhiBp)zNcerh{NlBG$Td*79N|~x&UFx0de{u zsZ*4txW!ET3zs+&vIaya7`)$OBf0r`(-mCQ)Mf zmJM_D3Ux-><60wVjoGfpiKmLEdw=fm^-Y5={oL?(2$o_BE&)p^r8LsqKqHTAh$ufC zkq_4zoUS3B4xQWF#_I3I?TMzyV=6dOd?ZR6z*zn#B|H1vvSkSj<<0Wmv7R1j<)+Id z_S=?`LD9a9H%K@kO3%eIOUq^tJd%`;5dAOmyWuyHvxRKgy!`EaIyUU4aTiCYo5J>o zami?{8Fo2EeJ{C)GiZ~&fQ=FFn;Y`Vb&+;2CieZ5`LM}HitOcW z)TCp$I6BJ)N>ooVe4J}pR2uAYRsH(fB>Mr(Q-K4DA*io-+u1zNd!-~&ITj9s5Rlo; z_YK#93uv#egaebKSH3AuBe9ZFp1`NW`3kqggL6}&YwbkH&#B3sq#)<+hw7eho}I)S zt!rx+niK*66~e-n(xQOoT}4`}2}-1$QGV$9ciuv6eS7md0yoJHj&S&wII^!r_OwMs z#2#v)eoBQ?`vO7L$zz?_gX1c>uGA?59)!W=d6gG|Syy1H+IETO%F_9VK+JEk+^>cH zs!uS$YVz{e!S7#oY4diK_m(^{Ad)XX(*XVQpDdgJ(PiYAwci9QhHUX)Ev!5#CwrxC zZ2)|jNF0q(DqWx}^J9al7*}m7A6iyKi-}?xPAe&&Ezd9-0u&I`@d# z;F7A5ImEC)^%lMP;Q?BqJXbLZ)@>3poV=RKLaf(V9~EnERqK$5p=ND9HZl2_c*-If z>|HkG46c}iwTKBML+na43=Ly+7AO?KSsU<$#3ZRCPOh&*66tedd1b9RR=MFx2^x%M zv5Yvmnc$4r1v{?S-zGK6y3@AB#i;d^rBd?YdoSdC4>%Z$%f^?;Bq6>9bj%HulbnXd zx-puMsLHX_#Qzd3}T@E^-HoQ*EbkT)<*><%#Zzs6&Sdq91Y8h3R0rO2s z0}wHC9qV(E%g@QR(}spLw;ba1+}5LQ_!V-=D4M}AsJSFwcTSbfNXf2LaWQwtRLFcA zvWz$fS#IY!Zt7{fY;JjT7+t5F&Ai=Kdz6#y9F~FTr-u8{!=7TQ#1lw!zO~$e1$fv7 z8C`a$olNQJ+DTn7arT&tkm<)(dMUcb8qJOCO!{7#xZ|U4S!^(6b|*iRq(P`-jfuhr zah&PKZv)d+-GDFl#O7+^4?FiOP#~lXb}bD?G~4RHl8H$%bY}^Q@QrS|c=;o}G$UkI zp43MO%l#s+jy%oIAx%dI(=HY4ns~PW#HGi2EvG$i^7qJxh!l}7m54{|4LPeih#9DD z3w^cM<3P~Prp9^|)g}XXWk4bUF<&lum_%>s+-@xJ_~S>l3CO(FQ@;K_&VGhG1OqzW z-oxHiSfH+1Qy}3~6Rx^oy%?JzUQT5niJ8)0XKteNTxyhfPCi$|1sG)~#n99y88>sa z8w+3t7y(Y9pbht{Z3>Tl3K?;dx~TA@}ooM)4h_O~#zz z(=c>r!EMQ|O~r{Xi3a!ZS<8q{H6t=rEZew^YK4m@wvHoWQhT$qayY?KZT1EfmKlBR z&Iy>C&N_&~<_}0u;mnLo@CZr9o>v+33~7h$_W^NzuD#WMc1TC+5iQ!wi!*!Z+6=ig zY=Z&Ell-)=s-hF4yFr-A9j;Km;%sdZ)IIOqkU4CKev@1vFi;GY%b}Tjit4uH6uq}3 z*c+C*{Dx_pIl})wE94L@^QpcNRo^Q`bHvm612ipyxt(*31Wk?sdl1>`J1ON%N!58P z9E|$bb*<*@*Dt#H@A{wC?@H0TjP~&jjRHlMe}Intmt~IX&nAyh@2HGqcIR<8Pp8Su zAf#N*YWk|y3U_R(E!?mDIP$?AOY(Ck?l(XefIZHC-b_L?wBaleYrS~eDS2rRoju?`+$^1tncX^^}aIgLP;ahy`tHrPHdn|V} zK@2?Vb$2alUR4ev)jg{mjGj!8v=9=7$o=uR+Jd>;GN+$hM8>}mh(I|b5_H3m8;h=Y zt8TWBgpTqOxN<_Piw)2T^tGnilVx5zdG)-eq5jc9k{-p1mV}(+dY}E|*!FK6lx??W z1Z+FjM(ItOalWyG=FLQPipOX_f_3QZlEU_>Y%ST;-`7aBCdr;4jGVR#v6tiIj1nY9 z`ae!j3Z4)q?#m5`WJN5z4kDR$Bw%k)lti{OCtKe+y+SsDcXwL(!KbH0&7NU@(1Y07 zW5d$x&Z}qeCwtS9G6((ujpgyeMhQ2mm^-NB-3}agE6(*`?d|u>(nMm|wye7S7|XTE zxoYGN1k&?Eh$FZF1JLSIt0tVHfcGGkHpw6d`c+Hf9_S~ z1=`+QS()AEt&Cct{I;DbbH*68d8YkghC*;m-8SdmBfc(%STDwa-qOez@*kj0Q-3QJ zg!^f0O*CN0SRe}*!wsoxuQ>QJxOm-69XhnFaP5LFe^oaoO`JSQ!d5j zNN}$F07d6Q)53{-UVt%Ni(uy04S1}hVwRQJ%lB<QrIOQws8H z0AE*~G;)>(EQO+(+Yj!Uu@n=q`^1~ZdR-gsbkMA4(Y7cmuuvT6*}nLZI@GXmdKr!1 zKT#{E8p!ClbgE2I~qHhYlk5{ADhm;Z{L7RgXEZ6)amk%Gw#O^<-0+B>YZm3~$Jf{)YFNN0N?31;%G5GqSD zl#WkJl$&oyN`s?R{S zurIph%G~z@bR{YjVS)>Mfk}jIv09!@pErZY&CUg8sk#q`5Wc$C%p8~X3Nm|FU03g| z(Eg3EQX%DoeMLo~j!{2uSNXz_Kn;_R@J3mn*p1}lw|Ly>pq(X4U=cd$#W8LF+4f%-ld!s$Dv9YrVZhcoN?UDFeG5PEwH#@13n+| z+FVsfEyS2Vmcwp(frKYJJmi9~Yrn#J0~K(w_G*Lb@|04NB_%m?7CV(r4Q1V@pRWM_ zKydm~OWQq@3eO@wy}eZcG3?{fGW;aaq19eeYgTDC*Z ztsJ@5-&ugzdER9-FQ<-RJ?_aLU)4Htvn`yT%#G*td$ViO7Q~~^v3k&I=7G2;IRouO zA)JmUDpBT{Nr?^#{!5)2_C^qNb4umT;2#1EkgyxHq4m9y*6& z&dM(}bfdB?gt&W1lXIi?zO_hy5p6q+9@pi15$m+6s>5saZJsDJByyJlmM;dy9O27*S#*6oSf*lZ09jyA9el|w?DZ2yM*1g!RzBuV6uc5x8WR)u zxQn(dhS|-iQyQFwqy&76#zbzX48(uqS{XZWwpR1t3xuCWK>R780|7cqpif@EG zK^J=UP8jO8Gr}cprYDx%gfEQ#2K~H2#8N^%-|MDk__*VIO+PJyU`Nua1upK|;9H7S z1-=lhPSVS2d6E_OV#LOg^p&>EW|%-RR1?)t>V>&-OO6eex}?w+skn+y=bf87ny*v! zxkP9HJnb2_*LQpE!NEz>?a? zr#c(9D}q&lB*q#!!8iS`ew)JG@=sHV^TjT93J#BQOQW=w%_?9voN@4F7hUKc&#o%O z#kvAq+vamZ=w7T?MSVI_#pBq(u(zT%pbF&?a2Kh*CHg>!7*RT66H7TG#!?(cR<*RW z8kX0T;jG#v+6*9}y#)t!eH`H4(;q)>e!`aeFO~aL;KkYW;X_aYW}5~+MO|-Il_P}p z@Ai%)!nfqmkKJyghpX>@j-?e%5Qfi$d4}H-KF%xiavT48ylxH^?rnhzs;#+^TAM@N z+T@^$bx;ahnqnM22jzGsx{akM!H@3i;#W3(wf;)6VOL*^6Uius%wjYrTT)SzOXA1g z7A>bDy@brHGUXbMXL}0?(LX>Y1J*0OeWJZc9iIuMYEHUPYd>gxwrWJKw%AhAM@+U> zW*Nul(daI*J1Qp`WmT)Gh_x|g0sGN5j@aH%h$W2R}*Wku9>2kJ7; zy#Fk#-AgaoTH9#6Obf`Cdg-4m5xQBu&=OAuN-{iPc0Ba2eyr*(pAa~-yOa6j{W1bReJyP;XG3!FF8Kl_k3(Xk)Uk7Cs4$s#AZK8YYAZRY zxc~HkafeGwOU^S^e<#SzxPk7&S}u^pImr|DA~%g^ONx73dqZJSBIoSt=`{e!LbJTs z^50Gr^VX{1>F6)Lgu;GTG1tZ)_Oke%ur~AuO|FS=e725X6HSbbVO)leAC7W-8dbLX zvr9As{+Ik-dk^$P?7CXKfFJ2s6?~8U`a<)B=!j%pTV7(ICw$iHe1Z>c0X?IW`SWOti zG&n-6F^-}V;l=-jR_AhRF;jRM=50UL!PBz2>t-0w+vl!w-@7|ysK1XmV=C4PTlkg_ z7=?0rOJm%0&X`2$1Wzm-&Xx@aMP>%N)v>GWlMdXHPNdXaHHF=D{@uWAiXlZ&nl7fFmvX<>vLCG z!c^*8NI$WLp4UAT!*}6U2Cfx{%vBbk`^*1W{r^Yy85B0A)hZ@GKm^&ux}k`B(M~XN9+2C^rOiqQv*pWhTq$jY!sMuFsK8%yiaR*qQ+IZtTxos0h zGxIxleL`hrFYW8In$>i@Zd>CnGlH{ z3xCAuW;81gjiZ-4Yff9HJBf1%vKJ`6Z#I*13XY}re~c4~Fl+otWDRghb86RG)~V zqE6r3uW-6ReP>>?0VB5sc1M~hLt!X$1G1nCIKd2XtCJt~6YByHySa_0?Evb2dSz;1 znPxl5XdFvzNriw?&>H|ssQP+1a4e3UQHYg#1{=V$5~CV+k4#jgS)|z~gqcEQyo(Qj6q{BF&pDYTk%szU9S?fes>e{SX-RGJPCgCcxG}bm!&0UL^ z35NINcrQF)s5GZ3wkndt7!{c@rq?Cz;$jQlB%Snhq=P4FqHX724C@tDK0HYK@_D&< z3ZZNIWRiQdXx~j-NCvrdCP#yt+jmm)&wo{1q#j7W`F5*8LB$e~ck$lD*XQr;U(Oud zLpLJ@$O^$!*kSqW zbx1hm`QS2|mQ%7o#jGXitgT?arxH4@wrTY!SIAk*@cm%zdp8L+RWNg4&9&fP78&O) z{0jdbz`tu*GgmUbQs9AV6SH`QE-f7~-+)@FAew+#w3JOcoHhlqGteC&T$lTYdw&6c zM}bp-mnGW*X;Z=>CM(4CxHRN!y$VA#dUc95t}|&JDzpmNM>0f^Vv{@%`PX=Qoo#Ya zIf_XbA$C?VnZYduS(H>(yNsY>qVWntFsJ>~s&d1qak5U*c0abeYB`)}si?imvaQJp zwIt@d(Dp2Q`-Oz5Es;yNoqIG&)-*)L`h>^yHo677K3)-AvQC+=n3^HvuHkw6IxWR$ zZEbLGYr=?XCBPXZ@;W+}L>Z^ouS-VWC}A7AfK9N}(pICMuGhsgO;+b*m@130R|y=7 z9J8HZZ4=p&L)ga^>mMKBVWCu|HozqXGM3$LREZRJ7mgD)w8q7i3eiQSyOuW3>qc+0 zOxt%QW^5W5)STP|hJg04w(soiFjFk?C+6&rAoJh%;qRzJ0XeykxLUWOz!(?6JiP67 zq!NgTm_tuag)Z*M>;BmTyta=3-&Qv@QIxGznMVHukjC#?ocEGXB~(wYyIY=$sGMAN z|8>+P3qzx%y@o5tI> zS}2u`qKvn1sPBi0%RIDc6D{k@?t2v^WKEVjgx_^~Ikca8rS z&m!D^AWGR5EYZI?IcUfR))tkMH@fvoO2GO7NPz~BrCY>#4^t%aI--NNL3n_CM~`+y zm@lX-JmS6U_MtcE%9!r5nwPD*?k?S)t3EY-xpX( z;ZVlpqNSpwOKf$uMX#EskadbHZ`(cR6<`y=GnCWcE^4F9 z%ZdF%Tn{aE-MW9E6gSFNhP3caNR17u3y575v$#=z-T5Lg66m^i9<_-l@vf7RM`17V z$?|^Rm;V`fBOt8CrtUh_(J)7UoGIwXQIR!MC%yiF7haW`J_aUy>66znE7$YoeQ^Dnw#D?0(NYp8-(I{Mt zUrqOMyvEB5FT-}or!OWmsC@$ZbhUy0>hu$G(ADXhmIsNf-{l9szPs%HxJ%Dw9*I!z z7Ig|JE~F~66Q~CQ`!je zWeymXMj}*`#I2Qm*HYHv0)2>lrDHT#I@b!f?G`ye7ubH6{2l+B{(sMlk+?M)Op)!A zUZ#)|;@u@1)9&HefcnH&Pzg8n+?!%QeX{f9tk+-t!;L;iy3%o^sP(v3$5w2X z$Hpb`4F_^-CCT#)UNdg)Q`J3I9%V~hE(=WyUu$poE}P6Ljfe2wnA;X}y*BKgQSURu zguXjC;Un!s@Kv!@yCCMidZ96Qp(DktaOp%ze?h;or@1C9J=|rfQHoN$K z_Csf1v8Zo-RpVR>ZXupUM%11sjoKgJ=RobBQ0?DW4{E736kfJVCGLzdb;dH9{)ONs zT6qZyCgEntjO;j-BKM~U&3*%2Bm^eSQq(Ai^*az?N%AuQezg2|a0(j&3B|`UV2fA5 z-6{l%PoZU%akv~@vSnuL%huvE!tLz4JGs24Pt;M*Uh5u%hoM7=1B)Ya=Bp_Hl(fVb| z+m}n`{ZHwfzH3#{hwN;vah^PfI4zAEi6*1yf$-kt_CJclfOEwG#X_W`4*9xf>1npA z9kUVDT$pPb?}_or88*wNIMThq z_}p=b=uhcj#)hYV>)Pe1))WtC5d#?R86M5=YW@*ZzYsFE(iYxGbQ)#xT3oQ_bBDC{ zsiPy(Zj>QsT^h#IbYg7WPE_YL^sD+O*Dc4fQ^BQriaFD1aRZy&TB+4>**sEo0#85u zC#VPDi34A}^v6$vtV_?{UGizW^o-!rPix65BHHYtU3XwG&%@OIajKLmvBka(!TV+W z7@Kb$x54WXV!*5=(5EXe5HZX&B>&i;n3)OQor`^sffZd-^;A+OLWyuq%@Ju`e$K8a zhH$Uh;(FN4b|g5q5`eJe)#be&tx5Ny&_!np1*@|E(wtAdj0T;Qw$=HTGw? zz!xFQ^i${u%rO*<6!XBe==A#}h@|+N1&r6SNR={4z}t|NBa}D~tOeHk^rl@HV)oD> zK0~j;KKR*nVUWUoilr~6fH~NGzyb3-{}a$vtn$UT#8=zs|Eddg_1q;%6say{>YPYm zgiO0Fdghw;z#jrtof_j_enh9%Nq1GRF}}q}^JaTZQBJ1;B+_Kh{@Iz|d3FCLsm{+H zh`bVEFi2m^Yw3R_a<7u&knhw;h(5g4L3S3qGh@U$MVpFV9H?f2pqtQ_p!>z6<`7+6QARxIG9Y|^R@BOU z({7NiN9#sbUR!!i11f_Lh5XjW&Rr02NSi?h@RnGBPf$+~Q`a34rj=9XGhEZFqAgM} z5HQPjB99N$tz16)Yo?N&dZC=~X30j&e%ETStYk3y^!p(QVM-CHZ)i1?57oc z(qu#T)V3B%hug(w6K6(#mR4W8AlL8kS_NR^>}%@UcipNEcoe($rDQu+>yLu0d^3+x zDK4v^E5P`?KZeXdNS;*zIRybp8^!Ut~X=j|{-<*ff!}dkDp6Ert zT~9r9T>AsmF7szBqzqdO=sIyRUIQ#}*nYe;iBFU2h%7KISEoMw=#z-=M${o>L5yx#TEN!->-iA>Q^4zXh^VHWTZZH*l`p$a zjI2K&aFKZE9tP~AUlemmN5&U+yGK-e%-B_>aRKLh*PjnHPchL`fD6A=gVVvQ>C_hSz-OY<;F$?@Q$@NZ*C$nay$R=}au^r6rz z^>(G(&oz$Yu__|$k{0}aEOXoiHTlgVq=!PCcQeM?`T$*SMUAPnMM1rFpVLDYU_t&{ zV5C>MAiDXf#S?zX-}xaqmY#_S{@~a7mXrJ6k)#(B9Rf6DxEl<-ugr(9N7@DV+STR%Yux7xwEjg}OCA{Zmvxh)q2MlkS|6<0t**H>h>aElxXMhO1NmTZ=@EShttK+qzgYm+g zjPHr+(8Hv_;s=WJ4-8sFG-`Q3(gE#%A3a*FH*Fo|ZX1Z%84_x_x~sb?jKx1dPrMc8 zZWpe+yucs_d6GQ?_(D^Wr)vd?0qmcR-S#oumSycovvWH_LfHfkB_oPFnN*Brw!QW0 zC;_yh1mRiy!0Wx|>HF0_cV9Psx$0?@T7=^A_ ztG3V5eOXn7mX#ZDjv!d7EZ9+-v7}NRccaI$WCX$;mf02_0q$0HKw_ah@i=dd&BRkI zwRL^3ptcuRZ@!RZ$%Ni~x515wn_phX3}>%%B#MkI0M^ehUJA4GTK&Nq`*5bCY{JqU z;Kr4Z#mS8A-r_B+XLm2!ZFirN&df462I0K}fmoz_58o;IqPOW6f13FHM~QTMudc{} zTz)`eBIDI*?hON?<-d8(=|qF@sFANj{j6G-`mua!i!Rj8C}374nYa(4aQi*lPXC9s z?|^G+dBP2%f(?W#AV^UG>7YR9#ReEK^iTtc5D`MJ(ySEeN{e(U34{_P^o}4c2nZqc z-U+=o@xG(CdH? z1`v@YP=vYQwpFfYb<}Ect!=w{;?l18VwZJtd>B4kZ99BS^4jx1*8o41(lIwd zvsxv}CpPG!5lMIrT3XsS^fV^0SQQusS6;RnHC&P5c6>;c(JrXpWx=Au%;}3;z1Nef zUC-ZUreOH`zn+=?X7WXojO-O5JCv@)uOKnZ%)Sx8r8Rh_z+a{!Z#;Rx_0g=PwM}Jc zdXO9Zvjp&K*tTlkwRwx}{amJ@N!6tR;YbU+ToW|FNfL8CY16Dlq> zjMNU6amQNq5_aCA-Ie=S{P6xMPFKf|iT`2Nu}gK$njBwS1)l$q{8**~05gyZ!>s7> z=b+r)hkbAT)T0ujX0Yh{9Y>%>Vyxy4)8juARl0m_z-@IIg&bbd}l0DT|<9)@g z4WAZ`NsBrDHeP?)_g94ivMH?X(;)4d1;Rb}A*K?m&(Q;=TG8XlVqBWG|6IO$+apTMWLGv0gqp5%LQjrq(1DVsJ9~om4IB z&a+jwBbBM}&ZCvuEU5ltK;-_j2yF6x@~aa7L#giD<_MQn+^zWbp5*>2WWCwy+pby9 zQz;Tl;Cn`Y}0J}Y@kJk2?iy>*K&GX6VXB40keMvAxB z14s98loqy9gCFK?o%}1v$eT{sgm=I0AhLs1CE1I|X+Ik9A z-8LxBAVOh8IJ;%Rw%uf#zRl3vsOgNCLi&1I&vM>J{epu|xy@t%3&L+bqjCppB#%%B zuk=7$^U2MgTN>BDtV)pYVKA-q(G2jIU`BFvyH8=ouM$sU%&Y)4A zPP|h&R~m}pRnZg?nRJwn?JncOLRLEEUi5>Gap@KXCo4*BZJB?4P8sz8bXIU@K)cu5 z7K^z_aq~BAlx!y9dEC7!i4O)TBZ|R}$?-?>s_urR;f*mI2|R_{#o4@=mJaN)s%$Sn z;Wn49b=sRBmMt>EKNNjf8ZbEb_7SGCk`Kl&Ldbh)GE{Hv(%NTQ6k*A(oKmuAmvabnaI8rg+>Uda@eA00LHcQa%V314Hbd6F}M1{l`YrO~+6H~rN8aXmI zZ`jvS%C6>XoZD%$bqSq%!$QeN=Eu3!>lo+mXir5!#M6eRQFN(X&4XrQpTCT;DFiSY z%^ohYlNxJ8Udr=2MGD2UU8YR92|Du|?z;8|?s{1FF+$&l47Rs!>d#0I7XXl!7Dbck z@a1Kw^4pKxx;i7tp3R3zm^ff11SEXW_%xXa=OW66E@KU<`Z%R6N*$P@!bl5Er>q+% zaz$x09)qqv|8d>9HobYqkWUAa-_E*g)bt(|f@L1OJ}EQxXD z=;Lm%z_wSdSZN{Yj0&yF6jl3P=~Rl?6B3+V{6_cm z>QeRyx2=BuiH2~oY5QA4eVax>fNM@9ke0|H;WF!ME`qEd4;R_&`9^-eJoe0pg)4iX~UlIIZSu_laH zMRkIsO?9TbEaLeVmb5~((^)r{3f~*3e&>7#fdc9xr8lOp-#h-{PZPgae)DTk$tMav z+B*ytnGpXPW-%n~dd>J6Br+Ll9ahwO&i|C6NSvDye519jHxCiJqFIc^(!iidl$ zL%i{c)vR?3r7S>v`~D}zv^>QEPCa~kkc$g*R3_2=JJ<=V>*MWJjAZYGNcomrPLX8q zo>IG85LPK$vwH=vciXu^E7Vlo=KWI1bs#VxAacKoAHAE)7NdXexJQ3ao=+t23cf|> z9P!D$X{|zIA>wB~I7*vjJ?BHJNU30>`E1U-z~IM8oz-Z5wT|xQPp166`p;W(0F|DA zYlZjGGDGKHT3i+-YW-2*uIjV5w2kp1BYfZPx`FK4_6n5_(TXjBlkH3^(E#15%yZV- z5OK2#ov*K-3U)nR(AeP2qWDj(>g5Adjj*O<4Y||^m~V3$MKM6D#uYjJhW+yXZfQ2) zbhh#(Es))fg+9PXMISr6r>hl$nXSD-3r83OYN3cP*%Dqv!w9Ot-PlTC1KA}?Re=a8 z=195oIdM^IDZ)sO1z0_t5|@{W&4J?ZnhAIfWP+-bFH=yZpOyVkF}d1K=1IM_o>H5$ z;l#8%I~12EX+3YOUJ;`m>hZ)?CD)4YTG0m(v!?I^VMKG$znFIa9`VeB^OT{6xbHrO zG-egEN|aqMSOgurvcV6}QK6N%w9Dq>>3bREY4RMzY|Jh#CSwFOk5kR}kJWv2W>dgvzT7vg?Q9vWbQ?ltDK-!;bd^s1V8Tr}rL%TQDlLnAlG{uxWcGa^k zX6Vq=jPx%sgjxi6on9?8OwHFo`*k~NCE*PqPo}U7$ehS8HXO7bB1;zi+pRl#N%?_C z>WP?=cn6qMc9uQ^xzU_a9OzaS>v?go@MKa?RiK;5Ra;G_-(=^4j~)FGT`9P84nZPf z67!7HdL=eTrrkV#H+;g~*ZB+i;kIK#i4@MTqjN1e;&DIuiwWCU1t3|X=@VHr9MQp>PVDPCy6>2O_g4@nN9kpmBX;&@TJvk02T}fJDNH|| zhhNxVcc#qmnwZ-La(*jjO=zSrU76ka@pCRerV^Oav*AsjML6>EobXnf(%vs9wg2FD zASd@N{rgLJPS+|3-~8*Gm2Y!$n#wG_p_45Cy~p~L|8Gxs9jTo5*(*PHMK`n6+|5cC z>F24$2)=&g()Kl?1?{F>E7?Qum7rQObCIC#Jk|UbLw(gT?2S_-!KW)s9cDP9 zDwQXqXE--)uT4((wHKWsJz+G<$ceOoMmk6HFfp-1zv00E{0;zi9@j~^>h%0CL;kEW zn^4I;=sD#YYZv<1EO&E5TuW;@gsyYiWDs&CS&CcmEM2Jc)r>!#DSy5WMA0%;3M9-M zu`b(AxhaxGvPF4lk6EnEE2pTHn9St6MdZK0q(E6X8$wg0#U8%@W4iE{f}d;uOlk4+ zvmnrw+_wP+tQ+eV_Cs;~xQh*fXTM&|Z-q5n)zmhK)-hpebyPU35m~;X+jTz=PpA2^ z&D-;;b&OYhsw;i8Wg%nVFYJAZz}(ZHk89>+teL1RixUcA-D8;$%BbxV*buZ~A~P%& znz7!-H4#fW{%N1T%;Ic8K*_%KR$-FUm=&)|4hIcStI{(Z%cjT>@F^)dCzRIM& z1}2o%Qac;!ikS5ufFBCyca&T_KY~K$wWE+D<~5< zr2-ONYrMS)r+xJ;;G+V|vr_1sNh_0}E5i`)~9@q4~VPn&5k zzrV-73f0?wvqblZp~4~oNydiPE?(f@{ zzw0=*HmzADeY#gZ7sUDgFTMY^b}SZl#^T49J}-4nL!z&NvxuVgoQ(!z-L36_4T2TK z#BT^79Pv_#Ix1q_hH4|2g8Vgn`UloGz00xfciEg5$K8J9z5qZufScxfp?q-i9=G66 zEPy|xz&|}LZ3E;rFqFg%BWfo&Ajx3_L!(3=NfE?Ulwd7HUo4#8hP|j%VqD8eCT)#y@Jc=PnFtYV$0=z_!|_$vYIs;=oAv(k(>Z)Hu$Hk_L&ATIwifjpFG z8yfF8i&_*8$gBlcXhx9;Xg0ZLDp8SY+&W6;`xgHnw*2`>v33RT^og1a5FEEy;i6!L zv|McY!<+C&D~81H+%gj)^ra@J%_|&M$j(ib;7Fa)8(8NE?RjolFn-9?TzgJ9(*$<= zo!!fNp^I6KK!C;*dWr9`h5xDi^V|@^%vNsny3`WAA$haE4UKEcRO)rL8(bkbRJb4NO-NsR~C-i=VufrA0~mtRkgD(!AB8|>+(#z)9JC~)k)fSIyoHAqnFJxSS&TV6#XcnL2K>sgO|ASc??}% z>G4Rzd8j9pI@$7ZdaM=_rW@r-6CDa$HRs1yIn_NEa>kvBF)<@)g}n~Yb)+_ zQ4B1*RO|R*n^XBTUD-blF_rv)hcZ9(TGd%ku?dM@sf`L($`k-EU@05-v{!}pYsB6d zfM3_abvGMl;Da_I#}5=wn)e^jHXdB;6ZDAm#`I|m*JNAa^$q#ZsEa(;P{U>#yf!v^ zL?1NOq#58s`I_InUHU-!5Kwmq(95L(s7zrxa;0Pwv!`!!hzKf}ZxA*h%te#FfW)8w zVcrRIss04N=*TC8V|D0~zWc1p@qYRepBH);R#Qhzf_jo!t3boYH}KDY${g}ppc`h2 zYrn6vY72qPr2<*h{wx$RyB^lca*y^sFBlD_YHz1-94}AY=rLAA@QNr1riHT`e@>BT zV|kl*Wz}aHHBy&hqY2C-LZaG`jVS5VoJ5?et~wYpDfU#P{mIoFta|@Op@O#Z83N|bTUoOUIodE2<|5fOu zo9LVqqaT)+s9I~8}z|%*kGxP9nnt(olD4sa;nzH&M|J{z%UwZzkQ2=~sD!&9r zZb~CEk~G}o?G-0{!t1D5!Qu1+S+8D^%h|M|tymi|6{%9nNcv#p=b6}WSetvYJ#G55 zt~+J8Ol$8B`N8Bj?iL_+(9-49wW`-S$HcxoIO)=o_*XA~)n4AbIAR<*W@JJ(%|T=r z`uN)N<=gOWAU+G{eXPuW8IvrK4VFK6bg(mMqaC8zF z#t~($mU9>O+9lK%;0v&d88kujwiE9($zw&9BCcsiCtD6VG=>QPo;97-DV`DZqLNc* zyo6u66ra+pNU*}AFxa3c&nY=qpSUG!u&T9TxpMhKSp{ZS;gU}EUqNqLxY}-Vf=>Nq z^7k62^tANXAd1kRrJ)vY0Av6shvTuEEr8rStEb67!p0$53&1p~0hp$OUqQEB4`9Y) z7GoB(V=uH+UAytVE14l{5qCcPGmhzJ&A-+E3EA{dO%M(>`P1$9^L9yJltjiSJAy5`Q*#u{P@OVY=6Hx1Og*<3w>vD#T%#gUTaFMDeD+1RJ)TboOA zUrRUhh~Y|>!uOKF&dV3A7ZU~awC5J@2#Xxu%EZvhlSPtoUZxw@lU%W@gVi3pdl%JU zxD-nvORJ0RUfrSQ9SU+EGjTKq#sc|3vMcbe_smD9PLXYy;OC&{PCJ;q1@Q&_rbWwC zv-Z8Dz>vY^nt`SMS}T`lPYkvJH|qjc6Ii=x`%E7cDmF1d)L~6+=Ek4U)H9seDO$M* zv3q1Aabs}-OHPEdU#|ssK!iYL^HSEL{Fy3T_F>yl zLWRrR?t^8b7ESvXTlZ!;JlZ6=~PmZZ78W}t{ z=~n8Y#WrYVS+wRMC0!gl(Y68>fa@YdEkcznURcCj=spzv#%aMOQYbPhA)ie9D+un` zdGH=YuKV3&%Fezq0XNrq{-}@wd%Rlw%eBLf`6dRv^b6_KxHBI)n~9Pl^-1chBrNyQ zxmxO#71+bEP> z!y(A-G2VP6bd(V|=qo`qo=-}bKb|r@!?rc|@yVwx4W{BknxUnE=0RuV3Xo4g{=I%2 zy2n^yv{#F-RQC$g#jWv(`#&%3PyaawI`*O=X-Cxo$JDao6Mp(A(G$fG^88%UG-(X% zYh+_8SQrv3|8AT1Ht7StTB*S4Dc*TF!`yWWv+VCSn&*;Zlk(D2NDpTe-m^lES}`{@ zJBxTDlE{Jlg6SSGMk)xc$sfC3m?*f-MeU=7i`1|L$gp5*`^>2B!y%5qW2TW6H%~JE zIqm`$Hs#>x<7j=u6Ygzh_l5oSup|Nbu*K76o+&oUFr5Y0wUT6!uUhN($6)E{dZ}X# z9v(tnQ=y`RSy*ekkZEAC@*!hc#yd4Djp^2`n4xlD%$_8VaC5$gU%Z)MGFJ1qNY(#s zMgQkJ0V+V!s$Q-r6XrH~#=c4o$y2UtGIgv-eT0ZTrlj^$Ih!V)QRR9hvL#Ll@(=>< zN-XT+7TSXJ2`8UWF_4PQ9yA?M@=@STf(@Oqcn1-O@{j+l{Cl154cCkCj2ECcRN|9c zn`64H(KCzaiCH_#9U@;ob%}9Yd$IK(R$0CBlHpuwOP}r5e#vlxLxTLJtq!tyyCM9B z_73`0h}QkA3RtU?2H8x<)z1Gk-)xzkCz-`AKhPtyyZJ`!)xE|U1M%=_{xP%xRK|6l1Y*pOx?IdbxfPxS4~!pIP}Wd&)C4unTy*Gb^XD;%Wp6MQnwW( zO7Sx`LfOmZ*uCc0%{{*_O@4k`7h75N@ZPSs7}_sQ4D#Em2Cw+Z0x z`p`RECFf{`(GuxC! zYVhf+XBQ1`5q0XUt~ZQ@L>q{$(8`$o^g zXm;@#>~Pd@fuMQP``W$DofIMGK4#L1(+gU`(|BXIvBGbeH#vUD5t*Us%=QL$(>;_W z7e7BrPpYg0o&4i-Ofh3)vt>G%X$ZJKY`w8H5p~Ob^Aosk^`I*I=>st&(w{`AyozQP z6ekej+4&X;${dDVD=2J_9h)THpkd^XJabrsc_GkAoL|l=;jf4IkQC zuSK?nz3~j~6MtOjkBVioKg~HPiDol%(N#3A^aL_HS*Vjr%a;yypL8HC>DL`z^grC{ zicPO6C?_b?`v57--q@*kbRVz>WXnA)101YxPQUoEYzQ)HE`EO8^&dVk|Mm2as@gdO zYwzRqq@LD35n&Zg${-v2^nf&GGgkHN75|*;k%)L8O>%mC`vtCD-4W*vewi1a@@qBf z_Z%V`f5GJBls}@^X!{he_x!J}LboQbQ>L^;fA}Ffh7V4Jn_c^0Kru^pb&B`bQ+BSO zu$irjt2>l%5U$Kl)DK_cxcz7zi%D{7n5*k04nOJHadRp;eRv_UEbC75sytL)h+Dv% zfp;x9f}H-O{su&e3nfS|x%X@f$OQqSr{)5deabRGvZie=y70^QI8iq~Qz_j2$)Gy-k5!n<8CqMBZDD= zT=%|bZTMfxKP8O)UtaWko4*$CgfD$kJ1aHLe$kfcLEw-65xRfToBS7r6n_h0+9-w${PYd0qMzBOKZ$4}rjfkgA@A>S>^XT3i=^d9My~n%|F7?E#!3 zV#Ig!DvDPk=x;jzy}!$Ejvu>u9z^}-)|Hzeib%0>kOfxidzejY3jn-Sbo^}>`Onu- z+bKzm(o4;`#bagWCun=$F;?pEX;cvNhyZOsO^PaZJ7d>izZI4CTI=jBr*&Vb>Yvli zKXv@O3I)VPSPx#81-kYgbn+jLG<-DYGDIYSvpRn&qGzs`TOdaxmKo(n>o?r|czG4M zH?_^S{EYQ!CKDj?(A-m@8yf9fZzikaOV%>+sByklWd?9d{##knVNij5aZ*C5=9#+ieENkY27dQ=io zs5vGmSmpMBn8B*I^D=DEfCSI=Unu4dHxR5;4KgAJ7ZXJLyYtqbKIn0K6(FCgWF#wM zaum~K2PsA|eQQ=?c{g?~{ZvMzH;^9ToN^st!vE!QB2BmG7W(-gaGrnnq5QXU%H(A> zcllOZltB`RM(7|(vZS)#4c@-cLvr3m1a@soT(5+{s+>xIFQ#2*OH_zVlB8mr{*hL( zOg3&jnY$}Y!0~6#*I)g-ym?GI+^6*q!LjRN;g;WHKLkGi>9gTOyv?T3d7|XdbNlry z+<3Yoal1?YE~z5iEa%w#Z9OC!^-jA=&TxQQxh(6W`BiO*xii=EKNS8ik9zPn2Gn)_ zzcr;q!B!XdMg5DPeyfe*iy^VyScre4tgkCtk$x}o!&h>eb4A+a&FrjbkLNVds%7=q zR&plx%V=s=uvQ{>ljwY0jMuksN&HT(9wh`u^S%7zUkBzjAorJI65v)MHd;|3&u;Ui z6Fcu0ZDb3h3G$%1nz99Ku{`W;=nPu{+-Sp8P90%6;)}fxFw5V$5sEHvzVVbNbzbJ_ zRQbx${kcKUPWUGM7whRId(n0*-=v+rsn5;nS0Tjt9G8MtQhk=3{digF_}lhN4yA=f zE*^%=m4_ug<+(M!p6qE~y4V-#Jh2aHT?i&(Uvid4#QeA&zEhSEN&U+uvho^^jDkB}42y z9?QfL3$-sW9Ac5~-Q|J4^Hnb%IJnpFFzvKF)784b)0d+)u@+yi9>(Sw?RPrR*Sx}f>T^14xV*Mg z^T4yU%tlt!spAkCyPGV!tuv_{s}Q4P6l9ZbQjm;zeK638-`kKExlMdTTONSik@8l& z+6KuCTiL$Pke|4-PT8EuG8ds4UxyVlkW1sqSdcQT9<<(<~z@+RAMk>b+0owkj+waM`bB@ zVSOhv67{&39cQUx2Ej`1u-!%}LMz{lM3uJLmzo+e+<@uCr3DyTYK}g?o!x)XD?gwu zCv>uk<0^Y(U?_>n-@lF)JY|gSAtAox+l04wO{VpM8`+cNEHsSVWpA)LATL$5ltMCV zoRW&_og;X7=YZQ3$U?fhGe}>eyp6+CbSG}keV9|2P)^`0%gqdj-`u(|G$VL4Evp^> zDqh=1yOR-t_A^IgZ}SV-jMpgZnNaV7S}-aZq(0@}n|caI*ly_X2QV^cI#9`J*%8C8~5&&2wLhz+oe9qIn+0 zjcRnUI%d~pCDn2+p-G8G^0wPfe!WRelK5d$`+Xb!)8>dAw%*8&{QJ>(MXm z@or5|99Y&2$S86U+f8s-vn`dqxUE%5ZV*VW&470|r+B#T;rWL*l>=wjV`wF`> zYJIlseZu1Vcv z_oq_@h1FJAf(V_BQBj}K;zIy`C1qdU!F+!``=BIn6+BB9jwD7*#IjqO9D_rLpCBx^ zp0i)eRLk}apF%j7!YsmC=WuozrEG8E z9-mX4HL(pRguQ4i)WPgn8eTqi+$#UrP1eCM?YyNthk^~YI&?*xp~51x$y9Vrxx!ccvEdG8zcuDUx@yml zP@FB9=fC4s|E}@hpPfK_E--XC_WtJ8tLAz1mdy zb6F8awB&r&#Wg99M`%NMnp?YWjJd&$T=$&H(Na#%;Zyb zZzMqHJV5d=iNxJzOA7)N$LkT+RIYz+Jz+NadPOoU4q3byjpl7)y^t{~4-ZnQAi7)N zw8IxZLipqqK;B&JZfZoqUI8>W|6GYwl(C+aIIve;Ix>J;=#;qa-5sTBFug`qM8i&q zPV%VNw$pb_XRVrGtNZ@8%tfpd+47kO26kX3E_fFMRRNw%{*8@f>XoXx@qpAmOz?_u zbozyoEQKf63#(Dq)^;&spUk->(mzUgjr6?(TCHC>!(ft@Br)4NEN|**aFxMCt$S>~ zyMVv4QlL4vs@tl3#4~*HGp(5JHhovQmf9<4!b&9SE5C1ZjNg^0=2hMM}^Zp7_*_@>+i2kaq zAp(U;7%HvxG#RIcy$LoZn#$c5cU_nmh$Y*PCbGR9VU|ND-=noA@C&O$xZCI6Mmu= zMcH0@=hC*Tjct7rG8i5Q%cxz?M6c%Ck0d;qqr#3fzyIFrl^uX&EyuqLW;*;~s!?+8 zOMu&~D0?f$BNl511QWMr2S;!@xmUZJq^*!~ zdXNqgx(!cCGxfk{{IZqaBsPMSDV|&w6Y-eHyD)RHy=mVrq#nQ`q{a$;bsF1c^ziK% zo^a{#N-VN2rt7x$&;s+j(Ktt(E!u30c*o|Iq+O|&8|FKsr&(5oZe~_8&%+nL(~#D`NJ4Gec~+;B>Kg77M%Op_sS%Inq&#eZ0W zN`|vj7rAXii0=ldN2?(aM~Bo#DP!gxbX2yJfAInSNy8^cqxf9X{MPxE8ESoeD`YC1 z>G*T&{FTLJns04ZChlc364U7slqE9RBscz^*|~FF5sMZ%Qk<}F5ra-!-eNu(iw1D# zf}Z#sl67Wgjf|&341bOQK|92hmxh#|4bzuWc6Ns}-DqaTJ(3A0Y7uV_BBK-YVB%|u zrjBi7c8q;kG2wc2E&u!8=_;v20hO%qa82E|8>C$y;S_5u1zGPxIaVYg| z;L47_R_LQ1lBhSVY}?t~&E0sgxU;-Kf0#f6my+C#%L8vL)i!QoS0Hz)CQ9u+Uyy_W zn6$}>GYEdYg-Hb}sGc%8-ICpcJGxV{qC_UoWyRJBkG?1;EF|bAb0{O0UpVjo#xe17 z8j&*NSX6SK--(YXZ;>mbZ7t_+{pZTnwd9%E=M6<*vF^unK3ABR3~e-6j|-91vkY?3 zSg!T*IocJ`<#OH~CI&hI5gJSRiO1nOG>1*Lj^H`K&av~Ryd9Jf zpV9L^Ghh<49$6tUgYAO3c84T7cTb03?2!Kz^!SdBzeF{3^!m`m?)3bUr_zpI$~n1P zJk!TvIIkp=7%c<@#ue`aFGEZP4O~J-4|7_=QBoLu#I@KtgoKgaIPDs&SjHm^$R^GY|hNa5@t55{{s(C?Ge$cZuC zAw!9-O=fq(cS_we>|z|iN*U!=v~SoaOG*;-l(IG_%UB0};kZ5%jjRdkCaq{wmqdWW zvOx+4V>!b~&5z*S?KYK@vRaiSsumB$$53$?l{x$DzgYNwhhPlS?mAZYz4rf9PU zR1STQ8p)yzsRTTI!cvwpjae*UNEr;7HF`F%;~#4|@BKHkQ1@yNNTV4Xrk zp3{XaRfryCMkJ!*bA1d!UcII1dT^*7{hG+VJn?z}YhqMJDU zo;X<$ab1SDZ=qOB6Vu*~>7#08qoR74$=(^ted@N8Ga=f@XJGWl#Xg1_8YT)zsM-h! zsW8_uQE|&Gm}Q1-FXTB!5~f(EW+Jgnc^i_NXhzc%vUq?Cmpm@<4UmIgd&&g^Xmb@uhmRxAVK7OipaOgte z=K1a}I5)Kpb(3k2t{!t1rPKoURY;hoy1=Qd4JFeEC@-&!fRhHFoq+_%N?u+}YfP8B zG*Ck)tk&BVm1!eG>@@L@gCAIr|a8>rWIMc2i0>O`-`@g}GE z&s-T*$v5-1nhp@U!_k%>cBKcBd`;56Y%@~ zR+`3a`+>jHbW+NW0Qw#o%oWsnn?^%Q?V}|v2fmu&vE`~;IbYilQi?4V1{S(7QU6P6 zxEZ-d>Z$Tb8124vpPhDSX=||*IJ#w7DQJGe#3dS809k%zeOX2}kD;XLJghw5xaTSR zGo6}Ra5icG+<2LP@}N5Q*+B~H?NIrEX>YXD;oW-L&(wYIWE4I z+0atE4JRbGHaYgohs{b!HTNhjH+M(}`a*WpLi3CH=itmwExjq{MR>o%=}_&K%KQR! z5aA$zSH~`Vc}Q=iY*M1xeL2Si?!-kwLl>p41;s(r01u@wC@=PQb>(#IT(Rc|Ivky6 z=+zY6NbcMG>Z*K_?NdPyPH@7nZeCOt$u+aY!!}U526izrsIsqpWn$q zwWHjay|@(kj#Sd_Os4dpve8d#p57BO5!p`pQqu{Li}~mrMBEK*RK#XDWIz2pwJi)K z5@qK7HTWcjbHDs$;idXEs+6HV)#TRlj}vWlou5xD5NlzUsRI>mwhz~q6F!N}^aL$d z-nm!0DE=l%Z7_zuEn>LiUB#g!K05tuefFmRqz=APSssgtNg4_mu(PhhP?<34XJI5r z+`JHkxW3_j(wLRVlIKWxWte@f8i$8Tu$GCaS1jpFj$?s+$ZX+ft=SzDeKovzhfE;# zvc@HS!R@PuW>flh3WY`xRRtgQ#(@KvnS1u98)22nisf$U#HJaWJb|kvnW&Ghl&{p_ zxNwwiv6f-ea)D>E563U@*55>y`BoBQ^yR}^XQi4s`*M^pm;pnR>wYus8T#u_QI-?K zd4-;lZQCh5Gy+ya3MFwc6yL%epWq=>FwHfFFmLUq20M{xKg1L__iYI)^e!pOn`?R~f)igcehB!=;3a;i#L!N4lQB!Lm7#XHr77%^-7 zTKKKjbL)f&tc)N`XQLr%FiVBPkP8$~wUr>WVHSMg-WfX*GZY_m;!B{JAlwj<&_;OV?dYC<@^B#^_X zO4cf8PaQ?~>At0_b+nR2U{T^C(kpyJMulDM6_UO)T#okMT)}95yqmz|Xgx+fo%Qjr zAhbR|FOV#3J#pO_Dk-?3R4r*!S%qmKWF$bWq~|*>x+At0c6CZ5Zht#qup+6&4kx)+ z+f~T2USY9ElERm26)v-*P4kHz^thJ3^1M;n_KQ1DabS17sH)gS!H9F+O;2}}yo|F+ zBesvdZBuUvEmV*C+{bp=ED#^hv7{{%Gf(LQ2`dyrY451y<{O?WLrAIFMtL>}WcB6A z7F1v+wAG!bW4+wK#NBWjz1!eY0-a~`s3Wo{K1rMOvDpW40lWPH1;@R2Q-fFYEllXn zD6Cd#@Llthy)WoIT! z^Nlrx{bXOGaxxq4^=&Pjx>Qty9mPCA=M9i|cNht+KK2inWsNm@dNK%uk8jqL_>Da~ z*%BO8p{HIBW63UPGTtvLuSyahGudhyoO#z4MVV7=auH}}5kvZpO1W$|iu=wFegmoi zRjdkI-$V7#yzb9fZz=G9`rQjk`KpMo$4f~jPk4x1+nU4^*1EMRnp4CV5W9%YtM7Xl zN{(zAs1oL}i}zI18q;@86{>U!ZT#95xipxh-_$fGFuKqd)fpf|!D#k0MPEq|s?Ida zuPWsfE4C%Y=Wz#rc-v;5Pe5=Devq8#ig)o_F0G2OUb2^JARok$28P53gL&NdhNnk4 z0}>AF2>N8+fR}W5QGGD8v`U2F+XbWi{dteK9SNa`(v-6k2^zPg4`m8d%pPRb8g(5#r8dwy}(d0w{MRRUx& z()NC3=1c$gCIK;xjZT8VeS}>%M`zi@mTP)ySZShlLV+jy;g+i`k41i8{HJ8I3sk7g ziniY11i^3~SD;}hhG(xX3%Lc zi^&61Znc_*R@S&84d6rh(SkscF^^p{-nTc1bMdWY99_7}-XbX6c?qI{L z*BQv|La%Jugtmp`GV)#5=-Ue>7az8|#G0L#v*BUNsmZ&4BQqIe5dSQ{;nbG=UZXPq z4Fk1E>1T`!tlU($ha@Z1L_!unFZgl7FI+9S@FDi*Z_|upUHnFu|6OSQKi+-+hjVny zZ6Fd<98U3RtE79^8neEJ|I&*VLN)dY*H9T6_MPDjY6Q=k1Zis|K*&0bEy{9DYrEi6@#myp}h}@CR$wo0O8mV>hcazjg9i3~(@EA-Qa(-%@7P~U7a7Hor$*5<3;8h2##m4{Xl|KJ z1T$}!F69j8P#Jdx$#rxtv&?CCFS6ul84dAj1gwI42AAza3g@>GMtjJ7=BSN(V_NA% za;yl3(IR%{6Phg{ov7v}<24~%)<&69u+`glrdYVDlpT~BUE8-Ow{nIW7O~wqB+I%rq@A8Y;w@x z6d!MHF}o!!%RdJ8$bA`*Fm*d!;hBU?}+!eB?gB_uW;Brh3>ma@$i`qxb1d88}!~6@+8`msVZws zR@x(nLRTju-dc1-XX@K3(>Jo}k9D zFO5Qgm0(`?*YEG%A;A@gsbi`;ZID7u?waTb?2WB9P9+i#Do~ziY?*?3QRJ$lABhNzlA z3r(GikgKh3_QN?@^O_qD7fl~YxaSI_wG!f1vTYde1Uwwuz3K5P*d&$LP*<>fh{|@D zGq%Q)b3JMx%lA{M^%$;H7`}I1_C>qARSp$QyU<7W^9ID{Akiyl3hSGYmG7))RA$u> zAx@P|BeQ+pYvlfwyD?va?MkZc{rd6}i}UXZEz#_9~L)sF!$($WI#R zpWa&|SJ8NFml&p!jIo(+j?qz?sFw#63$J2X%8H?`eHAWw&^uB?91;ixk{@qOICjN9 zxV$0YVcqIU2AfhK`51xb7e8GT>x`Hqp98+eD#Sf()@Zb&v@)@bpqQgjE?@bJ)5N%A z-v4RuTBDLm*Dz&eW7)+cR$gc(HN_O0Gn#n;O|n8F6(jOSS8_s7yde^Hs3|pXUDPlX zEbl1cXriK_rK5P6WVo22sacYlrelF=*P&^h)0s2#W2W=-oIiVg`}>}?*Z00_@3q%@ z-|u-IeZRCqvzfHqD8Ed^=!(TWn&4_wM%Ps4@cJcvNgYGJduK(yT+0waH{D>QRjj+W zCaWu1HsQT`7Oj2ZHp|=f1;RVLhzR@jo89gJ$!fc3rD=t2@I`j-?UO5{mWKJUdBk4Te(s`3e0Cu|IO!(0ims7T*eos=4{rdbX zY21Pg89CAiNxVxX?~9oSrJZm|?hzJ4H9T-c{H^>@In=4J6V=xkSM!2Hzcq3FFaoml zeoM-%bsN5nPg3PF0L6FPS)oKxj#uX?GCAOS6AP?Ru65xZ_XfM&hWWrO4^{LAYEtxy z)X!T`PK}8k>TP528Ws1nnhG84&()aLH?LTP462Bh-S*QPqvLZVunOes1>-uBUVh$#9V{gzp+6^L}*g?+VDubOc z1ipnC|d`6)Vd}DZngq$8#2a zS17CVj@?+d@n)(PR}bxvuZM9Kx(F3ouHhps*YLO*gJ)%8o`Y~4-T$vX?T1aR8;k)b zjyGR=>hH%*-?Z#*uF5cU56eIQ1ox5s(zy*!(7=F-OKfF&SQd?4m6C=|X~8WzP5kU| z0DinxA-wE`pB|v)wiIa1NIC(L5+6KHTnXY>OKu__2Bf~&c@>_*tjU}&^tX_wMxC_A z$L{o(E9UzQj9A(40VS!O4l0Tl_3amFwsp#k-kDvK>EsWknTM^H#@P<-Wh0Rhxlg}c za5fT&z|Rs2>0wjhDJiWaUV<1p$c&z>KW>v}>1YY=Arz}HuZXmcAu@ZBTJbn<`V zP7l6oralsMpHOKwn6XRF>gBWqncGQ&^2=Y9K8RsBbH4L|XeQ$Yhe?RX+pp{E0e2Fs zkQ&ff&f75$om9Vqv#|tXiIMlf!G|*&U$idEHNR1Fu@VWd=C=1gC_H!b8TCvh+wCAkgR zA;Vm7L|F8LSNtRoot5xX*^!f0LtclV)65P#pdPC1kaKus_W-AUG#oHSGd8OYI5#aI zfF0B&QY3cF*qZTpVI5;kj?4&T$hT=@;EP~rEaw#-h`I~agdwZ-n{SHcsr zhOM``M&uoRB(wyYQaNQhNm$HFf&;~q5A=z7VcD9vJi)}D^V6FYt*_B6G*)ivb{T6=kTw`M77cOk$TWRGF;k(r1)>ua+Kw2uJxej+y^i%b zF_G;BBxN7St`Y_ji#P|~12)qDD{VwZr6O6-QA1uin$jtOXeo@Zjnt^^uI|`U4A)zU zYMTm<1Lf%SR3H~bROxmR;QTkSucr;M7e1yX3CB|W}l()FBrxbvjN|}Mgm|RcAN@xQ{ zc}IoR+*fv))s2wqZ`g>*AIQ~n_!$=l=;#T^lP4AOiV2O~v=fm|9W#}nLQUftzX|B~ zHw)1ni>kZNeE<8E|9WP%bvx2PYZZJ!*FqhHZ}xEtZlJCb>_g8!4GVj-Yx1jN%fw6^ z{@!(pXIl;e5x>USR%-E5Y8Bjk4a2QXO5Y8FS*M%V81nB0nYJL9A5Y_G-Jb167FgN6 zDsBA0U!I%Wa5)6Jdob{muKA0Z1g%6$sjQqcFPeps*#Bv&^B2Rq>#>k;afk`A(j$v?}zg6%}p`kjpX4*J8{Ji*YHHn$~@%M_c zp1nB;cXPNF^O5HR9v&_op*x8mCd*v6v9WzoKU(B7&#f8O`|1h|wyf` of the descriptor is set to `"se_atten"`, which will use DPA-1 structures. +* {ref}`rcut ` is the cut-off radius for neighbor searching, and the {ref}`rcut_smth ` gives where the smoothing starts. +* **{ref}`sel `** gives the maximum possible number of neighbors in the cut-off radius. It is an int. Note that this number highly effects the efficiency of training, which we usually use less than 200. (We use 120 for training 56 elements in [OC2M dataset](https://github.com/Open-Catalyst-Project/ocp/blob/main/DATASET.md)) +* The {ref}`neuron ` specifies the size of the embedding net. From left to right the members denote the sizes of each hidden layer from input end to the output end, respectively. If the outer layer is of twice size as the inner layer, then the inner layer is copied and concatenated, then a [ResNet architecture](https://arxiv.org/abs/1512.03385) is built between them. +* The {ref}`axis_neuron ` specifies the size of submatrix of the embedding matrix, the axis matrix as explained in the [DeepPot-SE paper](https://arxiv.org/abs/1805.09003) +* If the option {ref}`resnet_dt ` is set to `true`, then a timestep is used in the ResNet. +* {ref}`seed ` gives the random seed that is used to generate random numbers when initializing the model parameters. +* {ref}`attn ` sets the length of hidden vector during scale-dot attention computation. +* {ref}`attn_layer ` sets the number of layers in attention mechanism. +* {ref}`attn_mask ` determines whether to mask the diagonal in the attention weights and False is recommended. +* {ref}`attn_dotr ` determines whether to dot the relative coordinates on the attention weights as a gated scheme, True is recommended. + +### Fitting `"ener"` +DPA-1 only support `"ener"` fitting type, and you can refer [here](train-energy.md) for detail information. + +### Type embedding +DPA-1 only support models with type embeddings on. And the default setting is as follows: +```json +"type_embedding":{ + "neuron": [8], + "resnet_dt": false, + "seed": 1 + } +``` +You can add these settings in input.json if you want to change the default ones, see [here](train-se-e2-a-tebd.md) for detail information. + + +### Type map +For training a large systems, especially those with dozens of elements, the {ref}`type ` determines the element index of training data: +```json +"type_map": [ + "Mg", + "Al", + "Cu" + ] +``` +which should include all the elements in the dataset you want to train on. +## Data format +DPA-1 supports the standard data format, which is detailed in [data-conv.md](../data/data-conv.md) and [system.md](../data/system.md). +Note that in this format, only those frames with the same fingerprint(i.e. the number of atoms of different element) can be put together as a unit system. +This may lead to sparse frame number in those rare systems. + +An ideal way is to put systems with same total number of atoms together, which is the way we trained DPA-1 on [OC2M](https://github.com/Open-Catalyst-Project/ocp/blob/main/DATASET.md). +This system format, which is called `mixed_type`, is proper to put frame-sparse systems together and is slightly different from the standard one. +Take an example, a `mixed_type` may contain the following files: +``` +type.raw +type_map.raw +set.000/box.npy +set.000/coord.npy +set.000/energy.npy +set.000/force.npy +set.000/real_atom_types.npy +``` +This system contains `Nframes` frames with the same atom number `Natoms`, the total number of element types contained in all frames is `Ntypes`. Note that we put all the frames in one set `set.000`. Most files are the same as those in [standard format](../data/system.md), here we only list the distinct ones: + +ID | Property | File | Required/Optional | Shape | Description +---------- | -------------------------------- | ------------------- | -------------------- | ----------------------- | ----------- +/ | Atom type indexes (place holder) | type.raw | Required | Natoms | All zeros to fake the type input +type_map | Atom type names | type_map.raw | Required | Ntypes | Atom names that map to atom type contained in all the frames, which is unnecessart to be contained in the periodic table +type | Atom type indexes of each frame | real_atom_types.npy | Required | Nframes \* Natoms | Integers that describe atom types in each frame, corresponding to indexes in type_map + +With these edited files, one can put together frames with the same `Natoms`, instead of the same formula (like `H2O`). Note that this `mixed_type` format only supports `se_atten` descriptor. + +The API to generate or transfer to `mixed_type` format will be uploaded on [dpdata](https://github.com/deepmodeling/dpdata) soon for a more convenient experience. + +## Training example +Here we upload the AlMgCu example showed in the paper, you can download here: +[Baidu disk](https://pan.baidu.com/s/1Mk9CihPHCmf8quwaMhT-nA?pwd=d586); +[Google disk](https://drive.google.com/file/d/11baEpRrvHoqxORFPSdJiGWusb3Y4AnRE/view?usp=sharing). + + + + + diff --git a/examples/water/se_atten/input.json b/examples/water/se_atten/input.json new file mode 100644 index 0000000000..2d54eaa1b6 --- /dev/null +++ b/examples/water/se_atten/input.json @@ -0,0 +1,70 @@ +{ + "_comment": " model parameters", + "model": { + "type_map": ["O", "H"], + "descriptor" :{ + "type": "se_atten", + "sel": 120, + "rcut_smth": 0.50, + "rcut": 6.00, + "neuron": [25, 50, 100], + "resnet_dt": false, + "axis_neuron": 16, + "seed": 1, + "attn": 128, + "attn_layer": 2, + "attn_dotr": true, + "attn_mask": false, + "_comment": " that's all" + }, + "fitting_net" : { + "neuron": [240, 240, 240], + "resnet_dt": true, + "seed": 1, + "_comment": " that's all" + }, + "_comment": " that's all" + }, + + "learning_rate" :{ + "type": "exp", + "decay_steps": 5000, + "start_lr": 0.001, + "stop_lr": 3.51e-8, + "_comment": "that's all" + }, + + "loss" :{ + "type": "ener", + "start_pref_e": 0.02, + "limit_pref_e": 1, + "start_pref_f": 1000, + "limit_pref_f": 1, + "start_pref_v": 0, + "limit_pref_v": 0, + "_comment": " that's all" + }, + + "training" : { + "training_data": { + "systems": ["../data/data_0/", "../data/data_1/", "../data/data_2/"], + "batch_size": "auto", + "_comment": "that's all" + }, + "validation_data":{ + "systems": ["../data/data_3"], + "batch_size": 1, + "numb_btch": 3, + "_comment": "that's all" + }, + "numb_steps": 1000000, + "seed": 10, + "disp_file": "lcurve.out", + "disp_freq": 100, + "save_freq": 1000, + "_comment": "that's all" + }, + + "_comment": "that's all" +} + diff --git a/source/lib/include/neighbor_list.h b/source/lib/include/neighbor_list.h index 53e6d83d2c..0155715cdc 100644 --- a/source/lib/include/neighbor_list.h +++ b/source/lib/include/neighbor_list.h @@ -85,6 +85,17 @@ build_nlist_cpu( const int & mem_size, const float & rcut); +void use_nei_info_cpu( + int * nlist, + int * ntype, + bool * nmask, + const int * type, + const int * nlist_map, + const int nloc, + const int nnei, + const int ntypes, + const bool b_nlist_map); + #if GOOGLE_CUDA || TENSORFLOW_USE_ROCM /** *@brief Convert the a host memory InputNlist to a device memory InputNlist @@ -140,6 +151,17 @@ build_nlist_gpu( const int & mem_size, const float & rcut); +void use_nei_info_gpu( + int * nlist, + int * ntype, + bool * nmask, + const int * type, + const int * nlist_map, + const int nloc, + const int nnei, + const int ntypes, + const bool b_nlist_map); + #endif // GOOGLE_CUDA @@ -167,6 +189,17 @@ build_nlist_gpu_rocm( const int & mem_size, const float & rcut); +void use_nei_info_gpu_rocm( + int * nlist, + int * ntype, + bool * nmask, + const int * type, + const int * nlist_map, + const int nloc, + const int nnei, + const int ntypes, + const bool b_nlist_map); + #endif // TENSORFLOW_USE_ROCM } // namespace deepmd diff --git a/source/lib/include/prod_env_mat.h b/source/lib/include/prod_env_mat.h index 58f1ae8485..cab2cda93a 100644 --- a/source/lib/include/prod_env_mat.h +++ b/source/lib/include/prod_env_mat.h @@ -21,7 +21,8 @@ void prod_env_mat_a_cpu( const int nall, const float rcut, const float rcut_smth, - const std::vector sec); + const std::vector sec, + const int * f_type = NULL); template void prod_env_mat_r_cpu( @@ -60,7 +61,8 @@ void prod_env_mat_a_gpu_cuda( const int nall, const float rcut, const float rcut_smth, - const std::vector sec); + const std::vector sec, + const int * f_type=NULL); template void prod_env_mat_r_gpu_cuda( @@ -110,7 +112,8 @@ void prod_env_mat_a_gpu_rocm( const int nall, const float rcut, const float rcut_smth, - const std::vector sec); + const std::vector sec, + const int * f_type=NULL); template void prod_env_mat_r_gpu_rocm( diff --git a/source/lib/src/cuda/neighbor_list.cu b/source/lib/src/cuda/neighbor_list.cu index 100fcd6aca..3089ab956b 100644 --- a/source/lib/src/cuda/neighbor_list.cu +++ b/source/lib/src/cuda/neighbor_list.cu @@ -144,6 +144,58 @@ __global__ void map_nlist( } } +__global__ void map_nei_info( + int * nlist, + int * ntype, + bool * nmask, + const int * type, + const int * nlist_map, + const int nloc, + const int nnei, + const int ntypes +) +{ + int atom_idx=blockIdx.x; + int nei_idx=blockIdx.y*blockDim.y+threadIdx.y; + if(nei_idx>=nnei){return;} + int nlist_idx=atom_idx*nnei+nei_idx; + int nlist_item=nlist[nlist_idx]; + int temp=0; + if(nlist_item!=-1){ + temp=nlist_map[nlist_item]; + nlist[nlist_idx]=temp; + ntype[nlist_idx]=type[temp]; + nmask[nlist_idx]=true; + } + else{ + ntype[nlist_idx]=ntypes; + } +} + +__global__ void map_nei_info_noconvert( + int * nlist, + int * ntype, + bool * nmask, + const int * type, + const int nloc, + const int nnei, + const int ntypes +) +{ + int atom_idx=blockIdx.x; + int nei_idx=blockIdx.y*blockDim.y+threadIdx.y; + if(nei_idx>=nnei){return;} + int nlist_idx=atom_idx*nnei+nei_idx; + int nlist_item=nlist[nlist_idx]; + if(nlist_item!=-1){ + ntype[nlist_idx]=type[nlist_item]; + nmask[nlist_idx]=true; + } + else{ + ntype[nlist_idx]=ntypes; + } +} + namespace deepmd { template int build_nlist_gpu( @@ -220,6 +272,32 @@ void use_nlist_map( DPErrcheck(cudaDeviceSynchronize()); } +void use_nei_info_gpu( + int * nlist, + int * ntype, + bool * nmask, + const int * type, + const int * nlist_map, + const int nloc, + const int nnei, + const int ntypes, + const bool b_nlist_map) +{ + int nblock=(nnei+TPB-1)/TPB; + dim3 block_grid(nloc, nblock); + dim3 thread_grid(1, TPB); + DPErrcheck(cudaMemset(ntype, 0, sizeof(int) * nloc * nnei)); + DPErrcheck(cudaMemset(nmask, 0, sizeof(bool) * nloc * nnei)); + if (b_nlist_map){ + map_nei_info<<>>(nlist, ntype, nmask, type, nlist_map, nloc, nnei, ntypes); + } + else{ + map_nei_info_noconvert<<>>(nlist, ntype, nmask, type, nloc, nnei, ntypes); + } + DPErrcheck(cudaGetLastError()); + DPErrcheck(cudaDeviceSynchronize()); +} + template int build_nlist_gpu(InputNlist & nlist, int * max_list_size, int * nlist_data, const float * c_cpy, const int & nloc, const int & nall, const int & mem_size, const float & rcut); template int build_nlist_gpu(InputNlist & nlist, int * max_list_size, int * nlist_data, const double * c_cpy, const int & nloc, const int & nall, const int & mem_size, const float & rcut); } \ No newline at end of file diff --git a/source/lib/src/cuda/prod_env_mat.cu b/source/lib/src/cuda/prod_env_mat.cu index 93a2b6a787..a68067e949 100644 --- a/source/lib/src/cuda/prod_env_mat.cu +++ b/source/lib/src/cuda/prod_env_mat.cu @@ -610,8 +610,12 @@ void prod_env_mat_a_gpu_cuda( const int nall, const float rcut, const float rcut_smth, - const std::vector sec) + const std::vector sec, + const int * f_type) { + if (f_type == NULL){ + f_type = type; + } const int nnei = sec.back(); const int ndescrpt = nnei * 4; DPErrcheck(cudaMemset(em, 0, sizeof(FPTYPE) * int_64(nloc) * ndescrpt)); @@ -620,7 +624,7 @@ void prod_env_mat_a_gpu_cuda( format_nbor_list_gpu_cuda( nlist, - coord, type, gpu_inlist, array_int, array_longlong, max_nbor_size, nloc, nall, rcut, sec); + coord, f_type, gpu_inlist, array_int, array_longlong, max_nbor_size, nloc, nall, rcut, sec); nborErrcheck(cudaGetLastError()); nborErrcheck(cudaDeviceSynchronize()); @@ -688,8 +692,8 @@ void test_encoding_decoding_nbor_info_gpu_cuda( DPErrcheck(cudaDeviceSynchronize()); } -template void prod_env_mat_a_gpu_cuda(float * em, float * em_deriv, float * rij, int * nlist, const float * coord, const int * type, const InputNlist & gpu_inlist, int * array_int, unsigned long long * array_longlong, const int max_nbor_size, const float * avg, const float * std, const int nloc, const int nall, const float rcut, const float rcut_smth, const std::vector sec); -template void prod_env_mat_a_gpu_cuda(double * em, double * em_deriv, double * rij, int * nlist, const double * coord, const int * type, const InputNlist & gpu_inlist, int * array_int, unsigned long long * array_longlong, const int max_nbor_size, const double * avg, const double * std, const int nloc, const int nall, const float rcut, const float rcut_smth, const std::vector sec); +template void prod_env_mat_a_gpu_cuda(float * em, float * em_deriv, float * rij, int * nlist, const float * coord, const int * type, const InputNlist & gpu_inlist, int * array_int, unsigned long long * array_longlong, const int max_nbor_size, const float * avg, const float * std, const int nloc, const int nall, const float rcut, const float rcut_smth, const std::vector sec, const int * f_type); +template void prod_env_mat_a_gpu_cuda(double * em, double * em_deriv, double * rij, int * nlist, const double * coord, const int * type, const InputNlist & gpu_inlist, int * array_int, unsigned long long * array_longlong, const int max_nbor_size, const double * avg, const double * std, const int nloc, const int nall, const float rcut, const float rcut_smth, const std::vector sec, const int * f_type); template void prod_env_mat_r_gpu_cuda(float * em, float * em_deriv, float * rij, int * nlist, const float * coord, const int * type, const InputNlist & gpu_inlist, int * array_int, unsigned long long * array_longlong, const int max_nbor_size, const float * avg, const float * std, const int nloc, const int nall, const float rcut, const float rcut_smth, const std::vector sec); template void prod_env_mat_r_gpu_cuda(double * em, double * em_deriv, double * rij, int * nlist, const double * coord, const int * type, const InputNlist & gpu_inlist, int * array_int, unsigned long long * array_longlong, const int max_nbor_size, const double * avg, const double * std, const int nloc, const int nall, const float rcut, const float rcut_smth, const std::vector sec); template void format_nbor_list_gpu_cuda(int * nlist, const float * coord, const int * type, const deepmd::InputNlist & gpu_inlist,int * array_int,uint_64 * array_longlong,const int max_nbor_size,const int nloc, const int nall, const float rcut, const std::vector sec); diff --git a/source/lib/src/neighbor_list.cc b/source/lib/src/neighbor_list.cc index cae7630430..99362bcc08 100644 --- a/source/lib/src/neighbor_list.cc +++ b/source/lib/src/neighbor_list.cc @@ -820,6 +820,55 @@ build_nlist_cpu( return 0; } +void +deepmd:: +use_nei_info_cpu( + int * nlist, + int * ntype, + bool * nmask, + const int * type, + const int * nlist_map, + const int nloc, + const int nnei, + const int ntypes, + const bool b_nlist_map) +{ + if(b_nlist_map){ + for (int ii = 0; ii < nloc; ++ii){ + for (int jj = 0; jj < nnei; ++jj){ + int nlist_idx = ii*nnei+jj; + int record = nlist[nlist_idx]; + if (record >= 0){ + int temp = nlist_map[record]; + nlist[nlist_idx] = temp; + ntype[nlist_idx]=type[temp]; + nmask[nlist_idx]=true; + } + else{ + ntype[nlist_idx]=ntypes; + nmask[nlist_idx]=false; + } + } + } + } + else{ + for (int ii = 0; ii < nloc; ++ii){ + for (int jj = 0; jj < nnei; ++jj){ + int nlist_idx = ii*nnei+jj; + int record = nlist[nlist_idx]; + if (record >= 0){ + ntype[nlist_idx]=type[record]; + nmask[nlist_idx]=true; + } + else{ + ntype[nlist_idx]=ntypes; + nmask[nlist_idx]=false; + } + } + } + } +} + template int deepmd:: diff --git a/source/lib/src/prod_env_mat.cc b/source/lib/src/prod_env_mat.cc index 303542699c..d64e6a4b09 100644 --- a/source/lib/src/prod_env_mat.cc +++ b/source/lib/src/prod_env_mat.cc @@ -25,8 +25,12 @@ prod_env_mat_a_cpu( const int nall, const float rcut, const float rcut_smth, - const std::vector sec) + const std::vector sec, + const int * f_type) { + if (f_type == NULL){ + f_type = type; + } const int nnei = sec.back(); const int nem = nnei * 4; @@ -39,11 +43,11 @@ prod_env_mat_a_cpu( } // set type - std::vector d_type (nall); + std::vector d_f_type(nall); for (int ii = 0; ii < nall; ++ii) { - d_type[ii] = type[ii]; + d_f_type[ii] = f_type[ii]; } - + // build nlist std::vector > d_nlist_a(nloc); @@ -62,13 +66,13 @@ prod_env_mat_a_cpu( #pragma omp parallel for for (int ii = 0; ii < nloc; ++ii) { std::vector fmt_nlist_a; - int ret = format_nlist_i_cpu(fmt_nlist_a, d_coord3, d_type, ii, d_nlist_a[ii], rcut, sec); + int ret = format_nlist_i_cpu(fmt_nlist_a, d_coord3, d_f_type, ii, d_nlist_a[ii], rcut, sec); std::vector d_em_a; std::vector d_em_a_deriv; std::vector d_em_r; std::vector d_em_r_deriv; std::vector d_rij_a; - env_mat_a_cpu (d_em_a, d_em_a_deriv, d_rij_a, d_coord3, d_type, ii, fmt_nlist_a, sec, rcut_smth, rcut); + env_mat_a_cpu (d_em_a, d_em_a_deriv, d_rij_a, d_coord3, d_f_type, ii, fmt_nlist_a, sec, rcut_smth, rcut); // check sizes assert (d_em_a.size() == nem); @@ -77,10 +81,10 @@ prod_env_mat_a_cpu( assert (fmt_nlist_a.size() == nnei); // record outputs for (int jj = 0; jj < nem; ++jj) { - em[ii * nem + jj] = (d_em_a[jj] - avg[d_type[ii] * nem + jj]) / std[d_type[ii] * nem + jj]; + em[ii * nem + jj] = (d_em_a[jj] - avg[type[ii] * nem + jj]) / std[type[ii] * nem + jj]; } for (int jj = 0; jj < nem * 3; ++jj) { - em_deriv[ii * nem * 3 + jj] = d_em_a_deriv[jj] / std[d_type[ii] * nem + jj / 3]; + em_deriv[ii * nem * 3 + jj] = d_em_a_deriv[jj] / std[type[ii] * nem + jj / 3]; } for (int jj = 0; jj < nnei * 3; ++jj) { rij[ii * nnei * 3 + jj] = d_rij_a[jj]; @@ -175,7 +179,6 @@ prod_env_mat_r_cpu( } } - template void deepmd:: @@ -194,7 +197,8 @@ prod_env_mat_a_cpu( const int nall, const float rcut, const float rcut_smth, - const std::vector sec); + const std::vector sec, + const int * f_type); template void @@ -214,7 +218,8 @@ prod_env_mat_a_cpu( const int nall, const float rcut, const float rcut_smth, - const std::vector sec); + const std::vector sec, + const int * f_type); template void diff --git a/source/lib/src/rocm/neighbor_list.hip.cu b/source/lib/src/rocm/neighbor_list.hip.cu index 795bdd59d3..3bdfbee770 100644 --- a/source/lib/src/rocm/neighbor_list.hip.cu +++ b/source/lib/src/rocm/neighbor_list.hip.cu @@ -144,6 +144,58 @@ __global__ void map_nlist( } } +__global__ void map_nei_info( + int * nlist, + int * ntype, + bool * nmask, + const int * type, + const int * nlist_map, + const int nloc, + const int nnei, + const int ntypes +) +{ + int atom_idx=blockIdx.x; + int nei_idx=blockIdx.y*blockDim.y+threadIdx.y; + if(nei_idx>=nnei){return;} + int nlist_idx=atom_idx*nnei+nei_idx; + int nlist_item=nlist[nlist_idx]; + int temp=0; + if(nlist_item!=-1){ + temp=nlist_map[nlist_item]; + nlist[nlist_idx]=temp; + ntype[nlist_idx]=type[temp]; + nmask[nlist_idx]=true; + } + else{ + ntype[nlist_idx]=ntypes; + } +} + +__global__ void map_nei_info_noconvert( + int * nlist, + int * ntype, + bool * nmask, + const int * type, + const int nloc, + const int nnei, + const int ntypes +) +{ + int atom_idx=blockIdx.x; + int nei_idx=blockIdx.y*blockDim.y+threadIdx.y; + if(nei_idx>=nnei){return;} + int nlist_idx=atom_idx*nnei+nei_idx; + int nlist_item=nlist[nlist_idx]; + if(nlist_item!=-1){ + ntype[nlist_idx]=type[nlist_item]; + nmask[nlist_idx]=true; + } + else{ + ntype[nlist_idx]=ntypes; + } +} + namespace deepmd { template int build_nlist_gpu_rocm( @@ -221,6 +273,32 @@ void use_nlist_map( DPErrcheck(hipDeviceSynchronize()); } +void use_nei_info_gpu_rocm( + int * nlist, + int * ntype, + bool * nmask, + const int * type, + const int * nlist_map, + const int nloc, + const int nnei, + const int ntypes, + const bool b_nlist_map) +{ + int nblock=(nnei+TPB-1)/TPB; + dim3 block_grid(nloc, nblock); + dim3 thread_grid(1, TPB); + DPErrcheck(hipMemset(ntype, 0, sizeof(int) * nloc * nnei)); + DPErrcheck(hipMemset(nmask, 0, sizeof(FPTYPE) * nloc * nnei)); + if (b_nlist_map){ + hipLaunchKernelGGL(map_nei_info, block_grid, thread_grid, 0, 0, nlist, ntype, nmask, type, nlist_map, nloc, nnei, ntypes); + } + else{ + hipLaunchKernelGGL(map_nei_info_noconvert, block_grid, thread_grid, 0, 0, nlist, ntype, nmask, type, nloc, nnei, ntypes); + } + DPErrcheck(hipGetLastError()); + DPErrcheck(hipDeviceSynchronize()); +} + template int build_nlist_gpu_rocm(InputNlist & nlist, int * max_list_size, int * nlist_data, const float * c_cpy, const int & nloc, const int & nall, const int & mem_size, const float & rcut); template int build_nlist_gpu_rocm(InputNlist & nlist, int * max_list_size, int * nlist_data, const double * c_cpy, const int & nloc, const int & nall, const int & mem_size, const float & rcut); } \ No newline at end of file diff --git a/source/lib/src/rocm/prod_env_mat.hip.cu b/source/lib/src/rocm/prod_env_mat.hip.cu index 506a844a04..ce6227a7f8 100644 --- a/source/lib/src/rocm/prod_env_mat.hip.cu +++ b/source/lib/src/rocm/prod_env_mat.hip.cu @@ -608,8 +608,12 @@ void prod_env_mat_a_gpu_rocm( const int nall, const float rcut, const float rcut_smth, - const std::vector sec) + const std::vector sec, + const int * f_type) { + if (f_type == NULL){ + f_type = type; + } const int nnei = sec.back(); const int ndescrpt = nnei * 4; DPErrcheck(hipMemset(em, 0, sizeof(FPTYPE) * int_64(nloc) * ndescrpt)); @@ -618,7 +622,7 @@ void prod_env_mat_a_gpu_rocm( format_nbor_list_gpu_rocm( nlist, - coord, type, gpu_inlist, array_int, array_longlong, max_nbor_size, nloc, nall, rcut, sec); + coord, f_type, gpu_inlist, array_int, array_longlong, max_nbor_size, nloc, nall, rcut, sec); nborErrcheck(hipGetLastError()); nborErrcheck(hipDeviceSynchronize()); @@ -686,8 +690,8 @@ void test_encoding_decoding_nbor_info_gpu_rocm( DPErrcheck(hipDeviceSynchronize()); } -template void prod_env_mat_a_gpu_rocm(float * em, float * em_deriv, float * rij, int * nlist, const float * coord, const int * type, const InputNlist & gpu_inlist, int * array_int, unsigned long long * array_longlong, const int max_nbor_size, const float * avg, const float * std, const int nloc, const int nall, const float rcut, const float rcut_smth, const std::vector sec); -template void prod_env_mat_a_gpu_rocm(double * em, double * em_deriv, double * rij, int * nlist, const double * coord, const int * type, const InputNlist & gpu_inlist, int * array_int, unsigned long long * array_longlong, const int max_nbor_size, const double * avg, const double * std, const int nloc, const int nall, const float rcut, const float rcut_smth, const std::vector sec); +template void prod_env_mat_a_gpu_rocm(float * em, float * em_deriv, float * rij, int * nlist, const float * coord, const int * type, const InputNlist & gpu_inlist, int * array_int, unsigned long long * array_longlong, const int max_nbor_size, const float * avg, const float * std, const int nloc, const int nall, const float rcut, const float rcut_smth, const std::vector sec, const int * f_type); +template void prod_env_mat_a_gpu_rocm(double * em, double * em_deriv, double * rij, int * nlist, const double * coord, const int * type, const InputNlist & gpu_inlist, int * array_int, unsigned long long * array_longlong, const int max_nbor_size, const double * avg, const double * std, const int nloc, const int nall, const float rcut, const float rcut_smth, const std::vector sec, const int * f_type); template void prod_env_mat_r_gpu_rocm(float * em, float * em_deriv, float * rij, int * nlist, const float * coord, const int * type, const InputNlist & gpu_inlist, int * array_int, unsigned long long * array_longlong, const int max_nbor_size, const float * avg, const float * std, const int nloc, const int nall, const float rcut, const float rcut_smth, const std::vector sec); template void prod_env_mat_r_gpu_rocm(double * em, double * em_deriv, double * rij, int * nlist, const double * coord, const int * type, const InputNlist & gpu_inlist, int * array_int, unsigned long long * array_longlong, const int max_nbor_size, const double * avg, const double * std, const int nloc, const int nall, const float rcut, const float rcut_smth, const std::vector sec); template void format_nbor_list_gpu_rocm(int * nlist, const float * coord, const int * type, const deepmd::InputNlist & gpu_inlist,int * array_int,uint_64 * array_longlong,const int max_nbor_size,const int nloc, const int nall, const float rcut, const std::vector sec); diff --git a/source/lib/tests/test_env_mat_a_mix.cc b/source/lib/tests/test_env_mat_a_mix.cc new file mode 100644 index 0000000000..03d01fe01d --- /dev/null +++ b/source/lib/tests/test_env_mat_a_mix.cc @@ -0,0 +1,1055 @@ +#include +#include +#include "fmt_nlist.h" +#include "env_mat.h" +#include "prod_env_mat.h" +#include "neighbor_list.h" +#include "device.h" + +class TestEnvMatAMix : public ::testing::Test +{ +protected: + std::vector posi = {12.83, 2.56, 2.18, + 12.09, 2.87, 2.74, + 00.25, 3.32, 1.68, + 3.36, 3.00, 1.81, + 3.51, 2.51, 2.60, + 4.27, 3.22, 1.56 + }; + std::vector atype = {0, 1, 1, 0, 1, 1}; + std::vector f_atype = {0, 0, 0, 0, 0, 0}; + std::vector posi_cpy; +// std::vector atype_cpy; + std::vector f_atype_cpy; + int nloc, nall; + double rc = 6; + double rc_smth = 0.8; + SimulationRegion region; + std::vector mapping, ncell, ngcell; + std::vector sec_a = {0, 20}; + std::vector sec_r = {0, 0}; + std::vector nat_stt, ext_stt, ext_end; + std::vector> nlist_a, nlist_r; + std::vector> nlist_a_cpy, nlist_r_cpy; + std::vector> ntype; + int f_ntypes = 1; + int ntypes = 2; // this information normally comes from natoms or avg/std + int nnei = sec_a.back(); + int ndescrpt = nnei * 4; + std::vector expected_env = { + 1.02167, -0.77271, 0.32370, 0.58475, 0.99745, 0.41810, 0.75655, -0.49773, 0.12206, 0.12047, 0.01502, -0.01263, 0.10564, 0.10495, -0.00143, 0.01198, 0.03103, 0.03041, 0.00452, -0.00425, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, + 1.02167, 0.77271, -0.32370, -0.58475, 0.59220, 0.42028, 0.16304, -0.38405, 0.04135, 0.04039, 0.00123, -0.00880, 0.03694, 0.03680, -0.00300, -0.00117, 0.00336, 0.00327, 0.00022, -0.00074, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, + 0.99745, -0.41810, -0.75655, 0.49773, 0.59220, -0.42028, -0.16304, 0.38405, 0.19078, 0.18961, -0.01951, 0.00793, 0.13499, 0.12636, -0.03140, 0.03566, 0.07054, 0.07049, -0.00175, -0.00210, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, + 1.06176, 0.16913, -0.55250, 0.89077, 1.03163, 0.96880, 0.23422, -0.26615, 0.19078, -0.18961, 0.01951, -0.00793, 0.12206, -0.12047, -0.01502, 0.01263, 0.04135, -0.04039, -0.00123, 0.00880, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, + 1.06176, -0.16913, 0.55250, -0.89077, 0.66798, 0.34516, 0.32245, -0.47232, 0.13499, -0.12636, 0.03140, -0.03566, 0.10564, -0.10495, 0.00143, -0.01198, 0.03694, -0.03680, 0.00300, 0.00117, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, + 1.03163, -0.96880, -0.23422, 0.26615, 0.66798, -0.34516, -0.32245, 0.47232, 0.07054, -0.07049, 0.00175, 0.00210, 0.03103, -0.03041, -0.00452, 0.00425, 0.00336, -0.00327, -0.00022, 0.00074, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, 0.00000, + }; + std::vector expected_ntype = { + 1, 1, 0, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 0, 1, 0, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 0, 1, 0, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 1, 1, 1, 0, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 0, 1, 1, 0, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 0, 1, 1, 0, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + }; + std::vector expected_nmask = { + 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + }; + + void SetUp() override { + double box[] = {13., 0., 0., 0., 13., 0., 0., 0., 13.}; + region.reinitBox(box); + copy_coord(posi_cpy, f_atype_cpy, mapping, ncell, ngcell, posi, f_atype, rc, region); + nloc = posi.size() / 3; + nall = posi_cpy.size() / 3; + nat_stt.resize(3); + ext_stt.resize(3); + ext_end.resize(3); + for (int dd = 0; dd < 3; ++dd){ + ext_stt[dd] = -ngcell[dd]; + ext_end[dd] = ncell[dd] + ngcell[dd]; + } + build_nlist(nlist_a, nlist_r, posi, rc, rc, ncell, region); + build_nlist(nlist_a_cpy, nlist_r_cpy, posi_cpy, nloc, rc, rc, nat_stt, ncell, ext_stt, ext_end, region, ncell); + } + void TearDown() override { + } +}; + + +class TestEnvMatAMixShortSel : public ::testing::Test +{ +protected: + std::vector posi = {12.83, 2.56, 2.18, + 12.09, 2.87, 2.74, + 00.25, 3.32, 1.68, + 3.36, 3.00, 1.81, + 3.51, 2.51, 2.60, + 4.27, 3.22, 1.56 + }; + std::vector atype = {0, 1, 1, 0, 1, 1}; + std::vector f_atype = {0, 0, 0, 0, 0, 0}; + std::vector posi_cpy; +// std::vector atype_cpy; + std::vector f_atype_cpy; + int nloc, nall; + double rc = 6; + double rc_smth = 0.8; + SimulationRegion region; + std::vector mapping, ncell, ngcell; + std::vector sec_a = {0, 4}; + std::vector sec_r = {0, 0}; + std::vector nat_stt, ext_stt, ext_end; + std::vector> nlist_a, nlist_r; + std::vector> nlist_a_cpy, nlist_r_cpy; + std::vector> ntype; + int f_ntypes = 1; + int ntypes = 2; // this information normally comes from natoms or avg/std + int nnei = sec_a.back(); + int ndescrpt = nnei * 4; + std::vector expected_env = { + 1.02167, -0.77271, 0.32370, 0.58475, 0.99745, 0.41810, 0.75655, -0.49773, 0.12206, 0.12047, 0.01502, -0.01263, 0.10564, 0.10495, -0.00143, 0.01198, + 1.02167, 0.77271, -0.32370, -0.58475, 0.59220, 0.42028, 0.16304, -0.38405, 0.04135, 0.04039, 0.00123, -0.00880, 0.03694, 0.03680, -0.00300, -0.00117, + 0.99745, -0.41810, -0.75655, 0.49773, 0.59220, -0.42028, -0.16304, 0.38405, 0.19078, 0.18961, -0.01951, 0.00793, 0.13499, 0.12636, -0.03140, 0.03566, + 1.06176, 0.16913, -0.55250, 0.89077, 1.03163, 0.96880, 0.23422, -0.26615, 0.19078, -0.18961, 0.01951, -0.00793, 0.12206, -0.12047, -0.01502, 0.01263, + 1.06176, -0.16913, 0.55250, -0.89077, 0.66798, 0.34516, 0.32245, -0.47232, 0.13499, -0.12636, 0.03140, -0.03566, 0.10564, -0.10495, 0.00143, -0.01198, + 1.03163, -0.96880, -0.23422, 0.26615, 0.66798, -0.34516, -0.32245, 0.47232, 0.07054, -0.07049, 0.00175, 0.00210, 0.03103, -0.03041, -0.00452, 0.00425, + }; + std::vector expected_ntype = { + 1, 1, 0, 1, + 1, 1, 1, 0, + 0, 1, 0, 1, + 0, 1, 0, 1, + 0, 1, 1, 0, + 0, 1, 1, 0, + }; + std::vector expected_nmask = { + 1, 1, 1, 1, + 1, 1, 1, 1, + 1, 1, 1, 1, + 1, 1, 1, 1, + 1, 1, 1, 1, + 1, 1, 1, 1, + }; + + void SetUp() override { + double box[] = {13., 0., 0., 0., 13., 0., 0., 0., 13.}; + region.reinitBox(box); + copy_coord(posi_cpy, f_atype_cpy, mapping, ncell, ngcell, posi, f_atype, rc, region); + nloc = posi.size() / 3; + nall = posi_cpy.size() / 3; + nat_stt.resize(3); + ext_stt.resize(3); + ext_end.resize(3); + for (int dd = 0; dd < 3; ++dd){ + ext_stt[dd] = -ngcell[dd]; + ext_end[dd] = ncell[dd] + ngcell[dd]; + } + build_nlist(nlist_a, nlist_r, posi, rc, rc, ncell, region); + build_nlist(nlist_a_cpy, nlist_r_cpy, posi_cpy, nloc, rc, rc, nat_stt, ncell, ext_stt, ext_end, region, ncell); + } + void TearDown() override { + } +}; + + +TEST_F(TestEnvMatAMix, orig_cpy) +{ + std::vector fmt_nlist_a, fmt_nlist_r; + std::vector env, env_deriv, rij_a; + bool pbc = false; + for(int ii = 0; ii < nloc; ++ii){ + int ret = format_nlist_i_cpu(fmt_nlist_a, posi_cpy, f_atype_cpy, ii, nlist_a_cpy[ii], rc, sec_a); + EXPECT_EQ(ret, -1); + env_mat_a(env, env_deriv, rij_a, posi_cpy, f_ntypes, f_atype_cpy, region, pbc, ii, fmt_nlist_a, sec_a, rc_smth, rc); + EXPECT_EQ(env.size(), sec_a.back()*4); + EXPECT_EQ(env.size(), env_deriv.size()/3); + EXPECT_EQ(rij_a.size(), sec_a.back()*3); + for (int jj = 0; jj < sec_a.back(); ++jj){ + for (int dd = 0; dd < 4; ++dd){ + EXPECT_LT(fabs(env[jj*4+dd] - expected_env[ii*sec_a.back()*4 + jj*4 + dd]) , 1e-5); + } + } + // for (int jj = 0; jj < sec_a.back(); ++jj){ + // printf("%7.5f, %7.5f, %7.5f, %7.5f, ", env[jj*4+0], env[jj*4+1], env[jj*4+2], env[jj*4+3]); + // } + // printf("\n"); + } +} + +TEST_F(TestEnvMatAMix, orig_pbc) +{ + std::vector fmt_nlist_a, fmt_nlist_r; + std::vector env, env_deriv, rij_a; + bool pbc = true; + for(int ii = 0; ii < nloc; ++ii){ + int ret = format_nlist_i_fill_a(fmt_nlist_a, fmt_nlist_r, posi, f_ntypes, f_atype, region, pbc, ii, nlist_a[ii], nlist_r[ii], rc, sec_a, sec_r); + EXPECT_EQ(ret, -1); + env_mat_a(env, env_deriv, rij_a, posi, f_ntypes, f_atype, region, pbc, ii, fmt_nlist_a, sec_a, rc_smth, rc); + EXPECT_EQ(env.size(), sec_a.back()*4); + EXPECT_EQ(env.size(), env_deriv.size()/3); + EXPECT_EQ(rij_a.size(), sec_a.back()*3); + for (int jj = 0; jj < sec_a.back(); ++jj){ + for (int dd = 0; dd < 4; ++dd){ + EXPECT_LT(fabs(env[jj*4+dd] - expected_env[ii*sec_a.back()*4 + jj*4 + dd]) , 1e-5); + } + } + } +} + +TEST_F(TestEnvMatAMix, orig_cpy_equal_pbc) +{ + std::vector fmt_nlist_a_0, fmt_nlist_r_0; + std::vector fmt_nlist_a_1, fmt_nlist_r_1; + std::vector env_0, env_deriv_0, rij_a_0; + std::vector env_1, env_deriv_1, rij_a_1; + for(int ii = 0; ii < nloc; ++ii){ + int ret_0 = format_nlist_i_cpu(fmt_nlist_a_0, posi_cpy, f_atype_cpy, ii, nlist_a_cpy[ii], rc, sec_a); + EXPECT_EQ(ret_0, -1); + env_mat_a(env_0, env_deriv_0, rij_a_0, posi_cpy, f_ntypes, f_atype_cpy, region, false, ii, fmt_nlist_a_0, sec_a, rc_smth, rc); + int ret_1 = format_nlist_i_fill_a(fmt_nlist_a_1, fmt_nlist_r_1, posi, f_ntypes, f_atype, region, true, ii, nlist_a[ii], nlist_r[ii], rc, sec_a, sec_r); + EXPECT_EQ(ret_1, -1); + env_mat_a(env_1, env_deriv_1, rij_a_1, posi, f_ntypes, f_atype, region, true, ii, fmt_nlist_a_1, sec_a, rc_smth, rc); + EXPECT_EQ(env_0.size(), env_1.size()); + EXPECT_EQ(env_deriv_0.size(), env_deriv_1.size()); + EXPECT_EQ(rij_a_0.size(), rij_a_1.size()); + for (unsigned jj = 0; jj < env_0.size(); ++jj){ + EXPECT_LT(fabs(env_0[jj] - env_1[jj]), 1e-10); + } + for (unsigned jj = 0; jj < env_deriv_0.size(); ++jj){ + EXPECT_LT(fabs(env_deriv_0[jj] - env_deriv_1[jj]), 1e-10); + } + for (unsigned jj = 0; jj < rij_a_0.size(); ++jj){ + EXPECT_LT(fabs(rij_a_0[jj] - rij_a_1[jj]), 1e-10); + } + } +} + +TEST_F(TestEnvMatAMix, orig_cpy_num_deriv) +{ + std::vector fmt_nlist_a, fmt_nlist_r; + std::vector env, env_0, env_1, env_deriv, env_deriv_tmp, rij_a; + bool pbc = false; + double hh = 1e-5; + for(int ii = 0; ii < nloc; ++ii){ + int ret = format_nlist_i_cpu(fmt_nlist_a, posi_cpy, f_atype_cpy, ii, nlist_a_cpy[ii], rc, sec_a); + EXPECT_EQ(ret, -1); + env_mat_a(env, env_deriv, rij_a, posi_cpy, f_ntypes, f_atype_cpy, region, pbc, ii, fmt_nlist_a, sec_a, rc_smth, rc); + EXPECT_EQ(env.size(), sec_a.back()*4); + EXPECT_EQ(env.size(), env_deriv.size()/3); + EXPECT_EQ(rij_a.size(), sec_a.back()*3); + + for (int jj = 0; jj < sec_a.back(); ++jj){ + int j_idx = fmt_nlist_a[jj]; + if (j_idx < 0) continue; + for (int kk = 0; kk < 4; ++kk){ + for (int dd = 0; dd < 3; ++dd){ + std::vector posi_0 = posi_cpy; + std::vector posi_1 = posi_cpy; + posi_0[j_idx*3+dd] -= hh; + posi_1[j_idx*3+dd] += hh; + env_mat_a(env_0, env_deriv_tmp, rij_a, posi_0, f_ntypes, f_atype_cpy, region, pbc, ii, fmt_nlist_a, sec_a, rc_smth, rc); + env_mat_a(env_1, env_deriv_tmp, rij_a, posi_1, f_ntypes, f_atype_cpy, region, pbc, ii, fmt_nlist_a, sec_a, rc_smth, rc); + double num_deriv = (env_1[jj*4+kk] - env_0[jj*4+kk])/(2.*hh); + double ana_deriv = -env_deriv[jj*12+kk*3+dd]; + EXPECT_LT(fabs(num_deriv - ana_deriv), 1e-5); + } + } + } + // for (int jj = 0; jj < sec_a.back(); ++jj){ + // printf("%7.5f, %7.5f, %7.5f, %7.5f, ", env[jj*4+0], env[jj*4+1], env[jj*4+2], env[jj*4+3]); + // } + // printf("\n"); + } +} + +TEST_F(TestEnvMatAMix, cpu) +{ + std::vector fmt_nlist_a, fmt_nlist_r; + std::vector env, env_deriv, rij_a; + bool pbc = false; + for(int ii = 0; ii < nloc; ++ii){ + int ret = format_nlist_i_cpu(fmt_nlist_a, posi_cpy, f_atype_cpy, ii, nlist_a_cpy[ii], rc, sec_a); + EXPECT_EQ(ret, -1); + deepmd::env_mat_a_cpu(env, env_deriv, rij_a, posi_cpy, f_atype_cpy, ii, fmt_nlist_a, sec_a, rc_smth, rc); + EXPECT_EQ(env.size(), sec_a.back()*4); + EXPECT_EQ(env.size(), env_deriv.size()/3); + EXPECT_EQ(rij_a.size(), sec_a.back()*3); + for (int jj = 0; jj < sec_a.back(); ++jj){ + for (int dd = 0; dd < 4; ++dd){ + EXPECT_LT(fabs(env[jj*4+dd] - expected_env[ii*sec_a.back()*4 + jj*4 + dd]) , 1e-5); + } + } + } +} + +TEST_F(TestEnvMatAMix, cpu_equal_orig_cpy) +{ + std::vector fmt_nlist_a_0, fmt_nlist_r_0; + std::vector fmt_nlist_a_1, fmt_nlist_r_1; + std::vector env_0, env_deriv_0, rij_a_0; + std::vector env_1, env_deriv_1, rij_a_1; + for(int ii = 0; ii < nloc; ++ii){ + int ret_0 = format_nlist_i_cpu(fmt_nlist_a_0, posi_cpy, f_atype_cpy, ii, nlist_a_cpy[ii], rc, sec_a); + EXPECT_EQ(ret_0, -1); + env_mat_a(env_0, env_deriv_0, rij_a_0, posi_cpy, f_ntypes, f_atype_cpy, region, false, ii, fmt_nlist_a_0, sec_a, rc_smth, rc); + + int ret_1 = format_nlist_i_cpu(fmt_nlist_a_1, posi_cpy, f_atype_cpy, ii, nlist_a_cpy[ii], rc, sec_a); + + EXPECT_EQ(ret_1, -1); + deepmd::env_mat_a_cpu(env_1, env_deriv_1, rij_a_1, posi_cpy, f_atype_cpy, ii, fmt_nlist_a_1, sec_a, rc_smth, rc); + + EXPECT_EQ(env_0.size(), env_1.size()); + EXPECT_EQ(env_deriv_0.size(), env_deriv_1.size()); + EXPECT_EQ(rij_a_0.size(), rij_a_1.size()); + for (unsigned jj = 0; jj < env_0.size(); ++jj){ + EXPECT_LT(fabs(env_0[jj] - env_1[jj]), 1e-10); + } + for (unsigned jj = 0; jj < env_deriv_0.size(); ++jj){ + EXPECT_LT(fabs(env_deriv_0[jj] - env_deriv_1[jj]), 1e-10); + } + for (unsigned jj = 0; jj < rij_a_0.size(); ++jj){ + EXPECT_LT(fabs(rij_a_0[jj] - rij_a_1[jj]), 1e-10); + } + } +} + +TEST_F(TestEnvMatAMix, cpu_num_deriv) +{ + std::vector fmt_nlist_a, fmt_nlist_r; + std::vector env, env_0, env_1, env_deriv, env_deriv_tmp, rij_a; + bool pbc = false; + double hh = 1e-5; + for(int ii = 0; ii < nloc; ++ii){ + int ret = format_nlist_i_cpu(fmt_nlist_a, posi_cpy, f_atype_cpy, ii, nlist_a_cpy[ii], rc, sec_a); + EXPECT_EQ(ret, -1); + deepmd::env_mat_a_cpu(env, env_deriv, rij_a, posi_cpy, f_atype_cpy, ii, fmt_nlist_a, sec_a, rc_smth, rc); + EXPECT_EQ(env.size(), sec_a.back()*4); + EXPECT_EQ(env.size(), env_deriv.size()/3); + EXPECT_EQ(rij_a.size(), sec_a.back()*3); + + for (int jj = 0; jj < sec_a.back(); ++jj){ + int j_idx = fmt_nlist_a[jj]; + if (j_idx < 0) continue; + for (int kk = 0; kk < 4; ++kk){ + for (int dd = 0; dd < 3; ++dd){ + std::vector posi_0 = posi_cpy; + std::vector posi_1 = posi_cpy; + posi_0[j_idx*3+dd] -= hh; + posi_1[j_idx*3+dd] += hh; + env_mat_a(env_0, env_deriv_tmp, rij_a, posi_0, f_ntypes, f_atype_cpy, region, pbc, ii, fmt_nlist_a, sec_a, rc_smth, rc); + env_mat_a(env_1, env_deriv_tmp, rij_a, posi_1, f_ntypes, f_atype_cpy, region, pbc, ii, fmt_nlist_a, sec_a, rc_smth, rc); + double num_deriv = (env_1[jj*4+kk] - env_0[jj*4+kk])/(2.*hh); + double ana_deriv = -env_deriv[jj*12+kk*3+dd]; + EXPECT_LT(fabs(num_deriv - ana_deriv), 1e-5); + } + } + } + // for (int jj = 0; jj < sec_a.back(); ++jj){ + // printf("%7.5f, %7.5f, %7.5f, %7.5f, ", env[jj*4+0], env[jj*4+1], env[jj*4+2], env[jj*4+3]); + // } + // printf("\n"); + } +} + +TEST_F(TestEnvMatAMixShortSel, orig_cpy) +{ + std::vector fmt_nlist_a, fmt_nlist_r; + std::vector env, env_deriv, rij_a; + bool pbc = false; + for(int ii = 0; ii < nloc; ++ii){ + int ret = format_nlist_i_cpu(fmt_nlist_a, posi_cpy, f_atype_cpy, ii, nlist_a_cpy[ii], rc, sec_a); + EXPECT_EQ(ret, 0); + env_mat_a(env, env_deriv, rij_a, posi_cpy, f_ntypes, f_atype_cpy, region, pbc, ii, fmt_nlist_a, sec_a, rc_smth, rc); + EXPECT_EQ(env.size(), sec_a.back()*4); + EXPECT_EQ(env.size(), env_deriv.size()/3); + EXPECT_EQ(rij_a.size(), sec_a.back()*3); + for (int jj = 0; jj < sec_a.back(); ++jj){ + for (int dd = 0; dd < 4; ++dd){ + EXPECT_LT(fabs(env[jj*4+dd] - expected_env[ii*sec_a.back()*4 + jj*4 + dd]) , 1e-5); + } + } + // for (int jj = 0; jj < sec_a.back(); ++jj){ + // printf("%8.5f, %8.5f, %8.5f, %8.5f, ", env[jj*4+0], env[jj*4+1], env[jj*4+2], env[jj*4+3]); + // } + // printf("\n"); + } +} + +TEST_F(TestEnvMatAMixShortSel, orig_pbc) +{ + std::vector fmt_nlist_a, fmt_nlist_r; + std::vector env, env_deriv, rij_a; + bool pbc = true; + for(int ii = 0; ii < nloc; ++ii){ + int ret = format_nlist_i_fill_a(fmt_nlist_a, fmt_nlist_r, posi, f_ntypes, f_atype, region, pbc, ii, nlist_a[ii], nlist_r[ii], rc, sec_a, sec_r); + EXPECT_EQ(ret, 0); + env_mat_a(env, env_deriv, rij_a, posi, f_ntypes, f_atype, region, pbc, ii, fmt_nlist_a, sec_a, rc_smth, rc); + EXPECT_EQ(env.size(), sec_a.back()*4); + EXPECT_EQ(env.size(), env_deriv.size()/3); + EXPECT_EQ(rij_a.size(), sec_a.back()*3); + for (int jj = 0; jj < sec_a.back(); ++jj){ + for (int dd = 0; dd < 4; ++dd){ + EXPECT_LT(fabs(env[jj*4+dd] - expected_env[ii*sec_a.back()*4 + jj*4 + dd]) , 1e-5); + } + } + } +} + +TEST_F(TestEnvMatAMixShortSel, cpu) +{ + std::vector fmt_nlist_a, fmt_nlist_r; + std::vector env, env_deriv, rij_a; + bool pbc = false; + for(int ii = 0; ii < nloc; ++ii){ + int ret = format_nlist_i_cpu(fmt_nlist_a, posi_cpy, f_atype_cpy, ii, nlist_a_cpy[ii], rc, sec_a); + EXPECT_EQ(ret, 0); + deepmd::env_mat_a_cpu(env, env_deriv, rij_a, posi_cpy, f_atype_cpy, ii, fmt_nlist_a, sec_a, rc_smth, rc); + EXPECT_EQ(env.size(), sec_a.back()*4); + EXPECT_EQ(env.size(), env_deriv.size()/3); + EXPECT_EQ(rij_a.size(), sec_a.back()*3); + for (int jj = 0; jj < sec_a.back(); ++jj){ + for (int dd = 0; dd < 4; ++dd){ + EXPECT_LT(fabs(env[jj*4+dd] - expected_env[ii*sec_a.back()*4 + jj*4 + dd]) , 1e-5); + } + } + } +} + +TEST_F(TestEnvMatAMix, prod_cpu) +{ + EXPECT_EQ(nlist_r_cpy.size(), nloc); + int tot_nnei = 0; + int max_nbor_size = 0; + for(int ii = 0; ii < nlist_a_cpy.size(); ++ii){ + tot_nnei += nlist_a_cpy[ii].size(); + if (nlist_a_cpy[ii].size() > max_nbor_size){ + max_nbor_size = nlist_a_cpy[ii].size(); + } + } + std::vector ilist(nloc), numneigh(nloc); + std::vector firstneigh(nloc); + deepmd::InputNlist inlist(nloc, &ilist[0], &numneigh[0], &firstneigh[0]); + deepmd::convert_nlist(inlist, nlist_a_cpy); + + std::vector em(nloc * ndescrpt), em_deriv(nloc * ndescrpt * 3), rij(nloc * nnei * 3); + std::vector nlist(nloc * nnei); + std::vector ntype(nloc * nnei); + bool * nmask = new bool [nloc * nnei]; + memset(nmask, 0, sizeof(bool) * nloc * nnei); + std::vector avg(ntypes * ndescrpt, 0); + std::vector std(ntypes * ndescrpt, 1); + deepmd::prod_env_mat_a_cpu( + &em[0], + &em_deriv[0], + &rij[0], + &nlist[0], + &posi_cpy[0], + &atype[0], + inlist, + max_nbor_size, + &avg[0], + &std[0], + nloc, + nall, + rc, + rc_smth, + sec_a, + &f_atype_cpy[0]); + deepmd::use_nei_info_cpu( + &nlist[0], + &ntype[0], + nmask, + &atype[0], + &mapping[0], + nloc, + nnei, + ntypes, + true); + + for(int ii = 0; ii < nloc; ++ii){ + for (int jj = 0; jj < nnei; ++jj){ + for (int dd = 0; dd < 4; ++dd){ + EXPECT_LT(fabs(em[ii*nnei*4 + jj*4 + dd] - + expected_env[ii*nnei*4 + jj*4 + dd]) , + 1e-5); + } + EXPECT_EQ(ntype[ii*nnei+jj], expected_ntype[ii*nnei+jj]); + EXPECT_EQ(nmask[ii*nnei+jj], expected_nmask[ii*nnei+jj]); + } + } + free(nmask); +} + +TEST_F(TestEnvMatAMix, prod_cpu_equal_cpu) +{ + EXPECT_EQ(nlist_r_cpy.size(), nloc); + int tot_nnei = 0; + int max_nbor_size = 0; + for(int ii = 0; ii < nlist_a_cpy.size(); ++ii){ + tot_nnei += nlist_a_cpy[ii].size(); + if (nlist_a_cpy[ii].size() > max_nbor_size){ + max_nbor_size = nlist_a_cpy[ii].size(); + } + } + std::vector ilist(nloc), numneigh(nloc); + std::vector firstneigh(nloc); + deepmd::InputNlist inlist(nloc, &ilist[0], &numneigh[0], &firstneigh[0]); + convert_nlist(inlist, nlist_a_cpy); + std::vector em(nloc * ndescrpt), em_deriv(nloc * ndescrpt * 3), rij(nloc * nnei * 3); + std::vector nlist(nloc * nnei); + std::vector avg(ntypes * ndescrpt, 0); + std::vector std(ntypes * ndescrpt, 1); + deepmd::prod_env_mat_a_cpu( + &em[0], + &em_deriv[0], + &rij[0], + &nlist[0], + &posi_cpy[0], + &atype[0], + inlist, + max_nbor_size, + &avg[0], + &std[0], + nloc, + nall, + rc, + rc_smth, + sec_a, + &f_atype_cpy[0]); + + std::vector fmt_nlist_a_1, fmt_nlist_r_1; + std::vector env_1, env_deriv_1, rij_a_1; + for(int ii = 0; ii < nloc; ++ii){ + int ret_1 = format_nlist_i_cpu(fmt_nlist_a_1, posi_cpy, f_atype_cpy, ii, nlist_a_cpy[ii], rc, sec_a); + EXPECT_EQ(ret_1, -1); + deepmd::env_mat_a_cpu(env_1, env_deriv_1, rij_a_1, posi_cpy, f_atype_cpy, ii, fmt_nlist_a_1, sec_a, rc_smth, rc); + EXPECT_EQ(env_1.size(), nnei * 4); + EXPECT_EQ(env_deriv_1.size(), nnei * 4 * 3); + EXPECT_EQ(rij_a_1.size(), nnei * 3); + EXPECT_EQ(fmt_nlist_a_1.size(), nnei); + EXPECT_EQ(env_1.size() * nloc, em.size()); + EXPECT_EQ(env_deriv_1.size() * nloc, em_deriv.size()); + EXPECT_EQ(rij_a_1.size() * nloc, rij.size()); + EXPECT_EQ(fmt_nlist_a_1.size() * nloc, nlist.size()); + for (unsigned jj = 0; jj < env_1.size(); ++jj){ + EXPECT_LT(fabs(em[ii*nnei*4+jj] - env_1[jj]), 1e-10); + } + for (unsigned jj = 0; jj < env_deriv_1.size(); ++jj){ + EXPECT_LT(fabs(em_deriv[ii*nnei*4*3+jj] - env_deriv_1[jj]), 1e-10); + } + for (unsigned jj = 0; jj < rij_a_1.size(); ++jj){ + EXPECT_LT(fabs(rij[ii*nnei*3+jj] - rij_a_1[jj]), 1e-10); + } + for (unsigned jj = 0; jj < fmt_nlist_a_1.size(); ++jj){ + EXPECT_EQ(nlist[ii*nnei+jj], fmt_nlist_a_1[jj]); + } + } + + // for(int ii = 0; ii < nloc; ++ii){ + // for (int jj = 0; jj < nnei; ++jj){ + // for (int dd = 0; dd < 4; ++dd){ + // EXPECT_LT(fabs(em[ii*nnei*4 + jj*4 + dd] - + // expected_env[ii*nnei*4 + jj*4 + dd]) , + // 1e-5); + // } + // } + // } +} + + +#if GOOGLE_CUDA +TEST_F(TestEnvMatAMix, prod_gpu_cuda) +{ + EXPECT_EQ(nlist_r_cpy.size(), nloc); + int tot_nnei = 0; + int max_nbor_size = 0; + for(int ii = 0; ii < nlist_a_cpy.size(); ++ii){ + tot_nnei += nlist_a_cpy[ii].size(); + if (nlist_a_cpy[ii].size() > max_nbor_size){ + max_nbor_size = nlist_a_cpy[ii].size(); + } + } + assert(max_nbor_size <= GPU_MAX_NBOR_SIZE); + if (max_nbor_size <= 1024) { + max_nbor_size = 1024; + } + else if (max_nbor_size <= 2048) { + max_nbor_size = 2048; + } + else { + max_nbor_size = 4096; + } + std::vector ilist(nloc), numneigh(nloc); + std::vector firstneigh(nloc); + deepmd::InputNlist inlist(nloc, &ilist[0], &numneigh[0], &firstneigh[0]), gpu_inlist; + convert_nlist(inlist, nlist_a_cpy); + std::vector em(nloc * ndescrpt, 0.0), em_deriv(nloc * ndescrpt * 3, 0.0), rij(nloc * nnei * 3, 0.0); + std::vector nlist(nloc * nnei, 0); + std::vector ntype(nloc * nnei, 0); + bool * nmask = new bool [nloc * nnei]; + memset(nmask, 0, sizeof(bool) * nloc * nnei); + std::vector avg(ntypes * ndescrpt, 0); + std::vector std(ntypes * ndescrpt, 1); + + double * em_dev = NULL, * em_deriv_dev = NULL, * rij_dev = NULL; + bool * nmask_dev = NULL; + double * posi_cpy_dev = NULL, * avg_dev = NULL, * std_dev = NULL; + int * f_atype_cpy_dev = NULL, * atype_dev = NULL, * nlist_dev = NULL, * ntype_dev = NULL, * mapping_dev = NULL, * array_int_dev = NULL, * memory_dev = NULL; + uint_64 * array_longlong_dev = NULL; + deepmd::malloc_device_memory_sync(em_dev, em); + deepmd::malloc_device_memory_sync(em_deriv_dev, em_deriv); + deepmd::malloc_device_memory_sync(rij_dev, rij); + deepmd::malloc_device_memory_sync(posi_cpy_dev, posi_cpy); + deepmd::malloc_device_memory_sync(avg_dev, avg); + deepmd::malloc_device_memory_sync(std_dev, std); + deepmd::malloc_device_memory_sync(f_atype_cpy_dev, f_atype_cpy); + deepmd::malloc_device_memory_sync(atype_dev, atype); + deepmd::malloc_device_memory_sync(nlist_dev, nlist); + deepmd::malloc_device_memory_sync(ntype_dev, ntype); + deepmd::malloc_device_memory_sync(mapping_dev, mapping); + deepmd::malloc_device_memory_sync(nmask_dev, nmask, nloc * nnei); + deepmd::malloc_device_memory(array_int_dev, sec_a.size() + nloc * sec_a.size() + nloc); + deepmd::malloc_device_memory(array_longlong_dev, nloc * GPU_MAX_NBOR_SIZE * 2); + deepmd::malloc_device_memory(memory_dev, nloc * max_nbor_size); + deepmd::convert_nlist_gpu_device(gpu_inlist, inlist, memory_dev, max_nbor_size); + + deepmd::prod_env_mat_a_gpu_cuda( + em_dev, + em_deriv_dev, + rij_dev, + nlist_dev, + posi_cpy_dev, + atype_dev, + gpu_inlist, + array_int_dev, + array_longlong_dev, + max_nbor_size, + avg_dev, + std_dev, + nloc, + nall, + rc, + rc_smth, + sec_a, + f_atype_cpy_dev); + + deepmd::use_nei_info_gpu( + nlist_dev, + ntype_dev, + nmask_dev, + atype_dev, + mapping_dev, + nloc, + nnei, + ntypes, + true); + deepmd::memcpy_device_to_host(em_dev, em); + deepmd::memcpy_device_to_host(ntype_dev, ntype); + deepmd::memcpy_device_to_host(nmask_dev, nmask, nloc * nnei); + deepmd::delete_device_memory(em_dev); + deepmd::delete_device_memory(em_deriv_dev); + deepmd::delete_device_memory(rij_dev); + deepmd::delete_device_memory(nlist_dev); + deepmd::delete_device_memory(ntype_dev); + deepmd::delete_device_memory(nmask_dev); + deepmd::delete_device_memory(posi_cpy_dev); + deepmd::delete_device_memory(f_atype_cpy_dev); + deepmd::delete_device_memory(atype_dev); + deepmd::delete_device_memory(mapping_dev); + deepmd::delete_device_memory(array_int_dev); + deepmd::delete_device_memory(array_longlong_dev); + deepmd::delete_device_memory(avg_dev); + deepmd::delete_device_memory(std_dev); + deepmd::delete_device_memory(memory_dev); + deepmd::free_nlist_gpu_device(gpu_inlist); + + for(int ii = 0; ii < nloc; ++ii){ + for (int jj = 0; jj < nnei; ++jj){ + for (int dd = 0; dd < 4; ++dd){ + EXPECT_LT(fabs(em[ii*nnei*4 + jj*4 + dd] - + expected_env[ii*nnei*4 + jj*4 + dd]) , + 1e-5); + } + EXPECT_EQ(ntype[ii*nnei+jj], expected_ntype[ii*nnei+jj]); + EXPECT_EQ(nmask[ii*nnei+jj], expected_nmask[ii*nnei+jj]); + } + } + free(nmask); +} + + +TEST_F(TestEnvMatAMix, prod_gpu_cuda_equal_cpu) +{ + EXPECT_EQ(nlist_r_cpy.size(), nloc); + int tot_nnei = 0; + int max_nbor_size = 0; + for(int ii = 0; ii < nlist_a_cpy.size(); ++ii){ + tot_nnei += nlist_a_cpy[ii].size(); + if (nlist_a_cpy[ii].size() > max_nbor_size){ + max_nbor_size = nlist_a_cpy[ii].size(); + } + } + assert(max_nbor_size <= GPU_MAX_NBOR_SIZE); + if (max_nbor_size <= 1024) { + max_nbor_size = 1024; + } + else if (max_nbor_size <= 2048) { + max_nbor_size = 2048; + } + else { + max_nbor_size = 4096; + } + std::vector ilist(nloc), numneigh(nloc); + std::vector firstneigh(nloc); + deepmd::InputNlist inlist(nloc, &ilist[0], &numneigh[0], &firstneigh[0]), gpu_inlist; + convert_nlist(inlist, nlist_a_cpy); + std::vector em(nloc * ndescrpt, 0.0), em_deriv(nloc * ndescrpt * 3, 0.0), rij(nloc * nnei * 3, 0.0); + std::vector nlist(nloc * nnei, 0); + std::vector avg(ntypes * ndescrpt, 0); + std::vector std(ntypes * ndescrpt, 1); + + double * em_dev = NULL, * em_deriv_dev = NULL, * rij_dev = NULL; + double * posi_cpy_dev = NULL, * avg_dev = NULL, * std_dev = NULL; + int * f_atype_cpy_dev = NULL, * atype_dev = NULL, * nlist_dev = NULL, * array_int_dev = NULL, * memory_dev = NULL; + uint_64 * array_longlong_dev = NULL; + deepmd::malloc_device_memory_sync(em_dev, em); + deepmd::malloc_device_memory_sync(em_deriv_dev, em_deriv); + deepmd::malloc_device_memory_sync(rij_dev, rij); + deepmd::malloc_device_memory_sync(posi_cpy_dev, posi_cpy); + deepmd::malloc_device_memory_sync(avg_dev, avg); + deepmd::malloc_device_memory_sync(std_dev, std); + + deepmd::malloc_device_memory_sync(f_atype_cpy_dev, f_atype_cpy); + deepmd::malloc_device_memory_sync(atype_dev, atype); + deepmd::malloc_device_memory_sync(nlist_dev, nlist); + deepmd::malloc_device_memory(array_int_dev, sec_a.size() + nloc * sec_a.size() + nloc); + deepmd::malloc_device_memory(array_longlong_dev, nloc * GPU_MAX_NBOR_SIZE * 2); + deepmd::malloc_device_memory(memory_dev, nloc * max_nbor_size); + deepmd::convert_nlist_gpu_device(gpu_inlist, inlist, memory_dev, max_nbor_size); + + deepmd::prod_env_mat_a_gpu_cuda( + em_dev, + em_deriv_dev, + rij_dev, + nlist_dev, + posi_cpy_dev, + atype_dev, + gpu_inlist, + array_int_dev, + array_longlong_dev, + max_nbor_size, + avg_dev, + std_dev, + nloc, + nall, + rc, + rc_smth, + sec_a, + f_atype_cpy_dev); + deepmd::memcpy_device_to_host(em_dev, em); + deepmd::memcpy_device_to_host(em_deriv_dev, em_deriv); + deepmd::memcpy_device_to_host(rij_dev, rij); + deepmd::memcpy_device_to_host(nlist_dev, nlist); + deepmd::delete_device_memory(em_dev); + deepmd::delete_device_memory(em_deriv_dev); + deepmd::delete_device_memory(nlist_dev); + deepmd::delete_device_memory(posi_cpy_dev); + deepmd::delete_device_memory(f_atype_cpy_dev); + deepmd::delete_device_memory(atype_dev); + deepmd::delete_device_memory(array_int_dev); + deepmd::delete_device_memory(array_longlong_dev); + deepmd::delete_device_memory(avg_dev); + deepmd::delete_device_memory(std_dev); + deepmd::delete_device_memory(memory_dev); + deepmd::free_nlist_gpu_device(gpu_inlist); + + std::vector fmt_nlist_a_1, fmt_nlist_r_1; + std::vector env_1, env_deriv_1, rij_a_1; + for(int ii = 0; ii < nloc; ++ii){ + int ret_1 = format_nlist_i_cpu(fmt_nlist_a_1, posi_cpy, f_atype_cpy, ii, nlist_a_cpy[ii], rc, sec_a); + EXPECT_EQ(ret_1, -1); + deepmd::env_mat_a_cpu(env_1, env_deriv_1, rij_a_1, posi_cpy, f_atype_cpy, ii, fmt_nlist_a_1, sec_a, rc_smth, rc); + EXPECT_EQ(env_1.size(), nnei * 4); + EXPECT_EQ(env_deriv_1.size(), nnei * 4 * 3); + EXPECT_EQ(rij_a_1.size(), nnei * 3); + EXPECT_EQ(fmt_nlist_a_1.size(), nnei); + EXPECT_EQ(env_1.size() * nloc, em.size()); + EXPECT_EQ(env_deriv_1.size() * nloc, em_deriv.size()); + EXPECT_EQ(rij_a_1.size() * nloc, rij.size()); + EXPECT_EQ(fmt_nlist_a_1.size() * nloc, nlist.size()); + for (unsigned jj = 0; jj < env_1.size(); ++jj){ + EXPECT_LT(fabs(em[ii*nnei*4+jj] - env_1[jj]), 1e-10); + } + for (unsigned jj = 0; jj < env_deriv_1.size(); ++jj){ + EXPECT_LT(fabs(em_deriv[ii*nnei*4*3+jj] - env_deriv_1[jj]), 1e-10); + } + for (unsigned jj = 0; jj < rij_a_1.size(); ++jj){ + EXPECT_LT(fabs(rij[ii*nnei*3+jj] - rij_a_1[jj]), 1e-10); + } + for (unsigned jj = 0; jj < fmt_nlist_a_1.size(); ++jj){ + EXPECT_EQ(nlist[ii*nnei+jj], fmt_nlist_a_1[jj]); + } + } + + for(int ii = 0; ii < nloc; ++ii){ + for (int jj = 0; jj < nnei; ++jj){ + for (int dd = 0; dd < 4; ++dd){ + EXPECT_LT(fabs(em[ii*nnei*4 + jj*4 + dd] - + expected_env[ii*nnei*4 + jj*4 + dd]) , + 1e-5); + } + } + } +} +#endif //GOOGLE_CUDA + +#if TENSORFLOW_USE_ROCM +TEST_F(TestEnvMatAMix, prod_gpu_rocm) +{ + EXPECT_EQ(nlist_r_cpy.size(), nloc); + int tot_nnei = 0; + int max_nbor_size = 0; + for(int ii = 0; ii < nlist_a_cpy.size(); ++ii){ + tot_nnei += nlist_a_cpy[ii].size(); + if (nlist_a_cpy[ii].size() > max_nbor_size){ + max_nbor_size = nlist_a_cpy[ii].size(); + } + } + assert(max_nbor_size <= GPU_MAX_NBOR_SIZE); + if (max_nbor_size <= 1024) { + max_nbor_size = 1024; + } + else if (max_nbor_size <= 2048) { + max_nbor_size = 2048; + } + else { + max_nbor_size = 4096; + } + std::vector ilist(nloc), numneigh(nloc); + std::vector firstneigh(nloc); + deepmd::InputNlist inlist(nloc, &ilist[0], &numneigh[0], &firstneigh[0]), gpu_inlist; + convert_nlist(inlist, nlist_a_cpy); + std::vector em(nloc * ndescrpt, 0.0), em_deriv(nloc * ndescrpt * 3, 0.0), rij(nloc * nnei * 3, 0.0); + std::vector nlist(nloc * nnei, 0); + std::vector ntype(nloc * nnei, 0); + bool * nmask = new bool [nloc * nnei]; + memset(nmask, 0, sizeof(bool) * nloc * nnei); + std::vector avg(ntypes * ndescrpt, 0); + std::vector std(ntypes * ndescrpt, 1); + + double * em_dev = NULL, * em_deriv_dev = NULL, * rij_dev = NULL, * nmask_dev = NULL; + double * posi_cpy_dev = NULL, * avg_dev = NULL, * std_dev = NULL; + int * f_atype_cpy_dev = NULL, * atype_dev = NULL, * nlist_dev = NULL, * ntype_dev = NULL, * mapping_dev = NULL, * array_int_dev = NULL, * memory_dev = NULL; + uint_64 * array_longlong_dev = NULL; + deepmd::malloc_device_memory_sync(em_dev, em); + deepmd::malloc_device_memory_sync(em_deriv_dev, em_deriv); + deepmd::malloc_device_memory_sync(rij_dev, rij); + deepmd::malloc_device_memory_sync(posi_cpy_dev, posi_cpy); + deepmd::malloc_device_memory_sync(avg_dev, avg); + deepmd::malloc_device_memory_sync(std_dev, std); + deepmd::malloc_device_memory_sync(f_atype_cpy_dev, f_atype_cpy); + deepmd::malloc_device_memory_sync(atype_dev, atype); + deepmd::malloc_device_memory_sync(nlist_dev, nlist); + deepmd::malloc_device_memory_sync(ntype_dev, ntype); + deepmd::malloc_device_memory_sync(mapping_dev, mapping); + deepmd::malloc_device_memory_sync(nmask_dev, nmask, nloc * nnei); + deepmd::malloc_device_memory(array_int_dev, sec_a.size() + nloc * sec_a.size() + nloc); + deepmd::malloc_device_memory(array_longlong_dev, nloc * GPU_MAX_NBOR_SIZE * 2); + deepmd::malloc_device_memory(memory_dev, nloc * max_nbor_size); + deepmd::convert_nlist_gpu_device(gpu_inlist, inlist, memory_dev, max_nbor_size); + + deepmd::prod_env_mat_a_gpu_rocm( + em_dev, + em_deriv_dev, + rij_dev, + nlist_dev, + posi_cpy_dev, + atype_dev, + gpu_inlist, + array_int_dev, + array_longlong_dev, + max_nbor_size, + avg_dev, + std_dev, + nloc, + nall, + rc, + rc_smth, + sec_a, + f_atype_cpy_dev); + + deepmd::use_nei_info_gpu_rocm( + nlist_dev, + ntype_dev, + nmask_dev, + atype_dev, + mapping_dev, + nloc, + nnei, + ntypes, + true); + deepmd::memcpy_device_to_host(em_dev, em); + deepmd::memcpy_device_to_host(ntype_dev, ntype); + deepmd::memcpy_device_to_host(nmask_dev, nmask, nloc * nnei); + deepmd::delete_device_memory(em_dev); + deepmd::delete_device_memory(em_deriv_dev); + deepmd::delete_device_memory(rij_dev); + deepmd::delete_device_memory(nlist_dev); + deepmd::delete_device_memory(ntype_dev); + deepmd::delete_device_memory(nmask_dev); + deepmd::delete_device_memory(posi_cpy_dev); + deepmd::delete_device_memory(f_atype_cpy_dev); + deepmd::delete_device_memory(atype_dev); + deepmd::delete_device_memory(mapping_dev); + deepmd::delete_device_memory(array_int_dev); + deepmd::delete_device_memory(array_longlong_dev); + deepmd::delete_device_memory(avg_dev); + deepmd::delete_device_memory(std_dev); + deepmd::delete_device_memory(memory_dev); + deepmd::free_nlist_gpu_device(gpu_inlist); + + for(int ii = 0; ii < nloc; ++ii){ + for (int jj = 0; jj < nnei; ++jj){ + for (int dd = 0; dd < 4; ++dd){ + EXPECT_LT(fabs(em[ii*nnei*4 + jj*4 + dd] - + expected_env[ii*nnei*4 + jj*4 + dd]) , + 1e-5); + } + EXPECT_EQ(ntype[ii*nnei+jj], expected_ntype[ii*nnei+jj]); + EXPECT_EQ(nmask[ii*nnei+jj], expected_nmask[ii*nnei+jj]); + } + } + free(nmask); +} + + +TEST_F(TestEnvMatAMix, prod_gpu_rocm_equal_cpu) +{ + EXPECT_EQ(nlist_r_cpy.size(), nloc); + int tot_nnei = 0; + int max_nbor_size = 0; + for(int ii = 0; ii < nlist_a_cpy.size(); ++ii){ + tot_nnei += nlist_a_cpy[ii].size(); + if (nlist_a_cpy[ii].size() > max_nbor_size){ + max_nbor_size = nlist_a_cpy[ii].size(); + } + } + assert(max_nbor_size <= GPU_MAX_NBOR_SIZE); + if (max_nbor_size <= 1024) { + max_nbor_size = 1024; + } + else if (max_nbor_size <= 2048) { + max_nbor_size = 2048; + } + else { + max_nbor_size = 4096; + } + std::vector ilist(nloc), numneigh(nloc); + std::vector firstneigh(nloc); + deepmd::InputNlist inlist(nloc, &ilist[0], &numneigh[0], &firstneigh[0]), gpu_inlist; + convert_nlist(inlist, nlist_a_cpy); + std::vector em(nloc * ndescrpt, 0.0), em_deriv(nloc * ndescrpt * 3, 0.0), rij(nloc * nnei * 3, 0.0); + std::vector nlist(nloc * nnei, 0); + std::vector avg(ntypes * ndescrpt, 0); + std::vector std(ntypes * ndescrpt, 1); + + double * em_dev = NULL, * em_deriv_dev = NULL, * rij_dev = NULL; + double * posi_cpy_dev = NULL, * avg_dev = NULL, * std_dev = NULL; + int * f_atype_cpy_dev = NULL, * atype_dev = NULL, * nlist_dev = NULL, * array_int_dev = NULL, * memory_dev = NULL; + uint_64 * array_longlong_dev = NULL; + deepmd::malloc_device_memory_sync(em_dev, em); + deepmd::malloc_device_memory_sync(em_deriv_dev, em_deriv); + deepmd::malloc_device_memory_sync(rij_dev, rij); + deepmd::malloc_device_memory_sync(posi_cpy_dev, posi_cpy); + deepmd::malloc_device_memory_sync(avg_dev, avg); + deepmd::malloc_device_memory_sync(std_dev, std); + + deepmd::malloc_device_memory_sync(f_atype_cpy_dev, f_atype_cpy); + deepmd::malloc_device_memory_sync(atype_dev, atype); + deepmd::malloc_device_memory_sync(nlist_dev, nlist); + deepmd::malloc_device_memory(array_int_dev, sec_a.size() + nloc * sec_a.size() + nloc); + deepmd::malloc_device_memory(array_longlong_dev, nloc * GPU_MAX_NBOR_SIZE * 2); + deepmd::malloc_device_memory(memory_dev, nloc * max_nbor_size); + deepmd::convert_nlist_gpu_device(gpu_inlist, inlist, memory_dev, max_nbor_size); + + deepmd::prod_env_mat_a_gpu_rocm( + em_dev, + em_deriv_dev, + rij_dev, + nlist_dev, + posi_cpy_dev, + atype_dev, + gpu_inlist, + array_int_dev, + array_longlong_dev, + max_nbor_size, + avg_dev, + std_dev, + nloc, + nall, + rc, + rc_smth, + sec_a, + f_atype_cpy_dev); + deepmd::memcpy_device_to_host(em_dev, em); + deepmd::memcpy_device_to_host(em_deriv_dev, em_deriv); + deepmd::memcpy_device_to_host(rij_dev, rij); + deepmd::memcpy_device_to_host(nlist_dev, nlist); + deepmd::delete_device_memory(em_dev); + deepmd::delete_device_memory(em_deriv_dev); + deepmd::delete_device_memory(nlist_dev); + deepmd::delete_device_memory(posi_cpy_dev); + deepmd::delete_device_memory(f_atype_cpy_dev); + deepmd::delete_device_memory(atype_dev); + deepmd::delete_device_memory(array_int_dev); + deepmd::delete_device_memory(array_longlong_dev); + deepmd::delete_device_memory(avg_dev); + deepmd::delete_device_memory(std_dev); + deepmd::delete_device_memory(memory_dev); + deepmd::free_nlist_gpu_device(gpu_inlist); + + std::vector fmt_nlist_a_1, fmt_nlist_r_1; + std::vector env_1, env_deriv_1, rij_a_1; + for(int ii = 0; ii < nloc; ++ii){ + int ret_1 = format_nlist_i_cpu(fmt_nlist_a_1, posi_cpy, f_atype_cpy, ii, nlist_a_cpy[ii], rc, sec_a); + EXPECT_EQ(ret_1, -1); + deepmd::env_mat_a_cpu(env_1, env_deriv_1, rij_a_1, posi_cpy, f_atype_cpy, ii, fmt_nlist_a_1, sec_a, rc_smth, rc); + EXPECT_EQ(env_1.size(), nnei * 4); + EXPECT_EQ(env_deriv_1.size(), nnei * 4 * 3); + EXPECT_EQ(rij_a_1.size(), nnei * 3); + EXPECT_EQ(fmt_nlist_a_1.size(), nnei); + EXPECT_EQ(env_1.size() * nloc, em.size()); + EXPECT_EQ(env_deriv_1.size() * nloc, em_deriv.size()); + EXPECT_EQ(rij_a_1.size() * nloc, rij.size()); + EXPECT_EQ(fmt_nlist_a_1.size() * nloc, nlist.size()); + for (unsigned jj = 0; jj < env_1.size(); ++jj){ + EXPECT_LT(fabs(em[ii*nnei*4+jj] - env_1[jj]), 1e-10); + } + for (unsigned jj = 0; jj < env_deriv_1.size(); ++jj){ + EXPECT_LT(fabs(em_deriv[ii*nnei*4*3+jj] - env_deriv_1[jj]), 1e-10); + } + for (unsigned jj = 0; jj < rij_a_1.size(); ++jj){ + EXPECT_LT(fabs(rij[ii*nnei*3+jj] - rij_a_1[jj]), 1e-10); + } + for (unsigned jj = 0; jj < fmt_nlist_a_1.size(); ++jj){ + EXPECT_EQ(nlist[ii*nnei+jj], fmt_nlist_a_1[jj]); + } + } + + for(int ii = 0; ii < nloc; ++ii){ + for (int jj = 0; jj < nnei; ++jj){ + for (int dd = 0; dd < 4; ++dd){ + EXPECT_LT(fabs(em[ii*nnei*4 + jj*4 + dd] - + expected_env[ii*nnei*4 + jj*4 + dd]) , + 1e-5); + } + } + } +} +#endif //TENSORFLOW_USE_ROCM diff --git a/source/op/prod_env_mat_multi_device.cc b/source/op/prod_env_mat_multi_device.cc index f88606053d..4e32904907 100644 --- a/source/op/prod_env_mat_multi_device.cc +++ b/source/op/prod_env_mat_multi_device.cc @@ -147,6 +147,36 @@ REGISTER_OP("DescrptSeR") .Output("rij: T") .Output("nlist: int32"); +REGISTER_OP("ProdEnvMatAMix") + .Attr("T: {float, double} = DT_DOUBLE") + .Input("coord: T") //atomic coordinates + .Input("type: int32") //atomic type + .Input("natoms: int32") //local atomic number; each type atomic number + .Input("box : T") + .Input("mesh : int32") + .Input("davg: T") //average value of data + .Input("dstd: T") //standard deviation + .Attr("rcut_a: float") //no use + .Attr("rcut_r: float") + .Attr("rcut_r_smth: float") + .Attr("sel_a: list(int)") + .Attr("sel_r: list(int)") //all zero + .Output("descrpt: T") + .Output("descrpt_deriv: T") + .Output("rij: T") + .Output("nlist: int32") + .Output("ntype: int32") + .Output("nmask: bool") + .Doc(R"(Compute the environment matrix mixing the atom types. +The sorting of neighbor atoms depends not on atom types, but on the distance and index. +The atoms in nlist matrix will gather forward and thus save space for gaps of types in ProdEnvMatA, +resulting in optimized and relative small sel_a. + +The additional outputs are listed as following: +ntype: The corresponding atom types in nlist. +nmask: The atom mask in nlist. +)"); + template static int _norm_copy_coord_cpu( @@ -184,6 +214,18 @@ _map_nlist_cpu( const int & nloc, const int & nnei); +static void +_map_nei_info_cpu( + int * nlist, + int * ntype, + bool * nmask, + const int * type, + const int * idx_mapping, + const int & nloc, + const int & nnei, + const int & ntypes, + const bool & b_nlist_map); + template static void _prepare_coord_nlist_cpu( @@ -252,6 +294,18 @@ _map_nlist_gpu( const int & nloc, const int & nnei); +static void +_map_nei_info_gpu( + int * nlist, + int * ntype, + bool * nmask, + const int * type, + const int * idx_mapping, + const int & nloc, + const int & nnei, + const int & ntypes, + const bool & b_nlist_map); + template static void _prepare_coord_nlist_gpu( @@ -326,6 +380,18 @@ _map_nlist_gpu_rocm( const int & nloc, const int & nnei); +static void +_map_nei_info_gpu_rocm( + int * nlist, + int * ntype, + bool * nmask, + const int * type, + const int * idx_mapping, + const int & nloc, + const int & nnei, + const int & ntypes, + const bool & b_nlist_map); + template static void _prepare_coord_nlist_gpu_rocm( @@ -869,7 +935,7 @@ class ProdEnvMatROp : public OpKernel { frame_nall, mem_cpy, mem_nnei, max_nbor_size, box, mesh_tensor.flat().data(), nloc, nei_mode, rcut, max_cpy_trial, max_nnei_trial); // launch the cpu compute function - prod_env_mat_r_cpu( + deepmd::prod_env_mat_r_cpu( em, em_deriv, rij, nlist, coord, type, inlist, max_nbor_size, avg, std, nloc, frame_nall, rcut, rcut_smth, sec); if(b_nlist_map) _map_nlist_cpu(nlist, &idx_mapping[0], nloc, nnei); @@ -896,7 +962,314 @@ class ProdEnvMatROp : public OpKernel { int * nbor_list_dev = NULL; }; +template +class ProdEnvMatAMixOp : public OpKernel { +public: + explicit ProdEnvMatAMixOp(OpKernelConstruction* context) : OpKernel(context) { + float nloc_f, nall_f; + OP_REQUIRES_OK(context, context->GetAttr("rcut_a", &rcut_a)); + OP_REQUIRES_OK(context, context->GetAttr("rcut_r", &rcut_r)); + OP_REQUIRES_OK(context, context->GetAttr("rcut_r_smth", &rcut_r_smth)); + OP_REQUIRES_OK(context, context->GetAttr("sel_a", &sel_a)); + OP_REQUIRES_OK(context, context->GetAttr("sel_r", &sel_r)); + // OP_REQUIRES_OK(context, context->GetAttr("nloc", &nloc_f)); + // OP_REQUIRES_OK(context, context->GetAttr("nall", &nall_f)); + deepmd::cum_sum (sec_a, sel_a); + deepmd::cum_sum (sec_r, sel_r); + ndescrpt_a = sec_a.back() * 4; + ndescrpt_r = sec_r.back() * 1; + ndescrpt = ndescrpt_a + ndescrpt_r; + nnei_a = sec_a.back(); + nnei_r = sec_r.back(); + nnei = nnei_a + nnei_r; + max_nbor_size = 1024; + max_cpy_trial = 100; + mem_cpy = 256; + max_nnei_trial = 100; + mem_nnei = 256; + } + + void Compute(OpKernelContext* context) override { + deepmd::safe_compute(context, [this](OpKernelContext* context) {this->_Compute(context);}); + } + + void _Compute(OpKernelContext* context) { + // Grab the input tensor + int context_input_index = 0; + const Tensor& coord_tensor = context->input(context_input_index++); + const Tensor& type_tensor = context->input(context_input_index++); + const Tensor& natoms_tensor = context->input(context_input_index++); + const Tensor& box_tensor = context->input(context_input_index++); + const Tensor& mesh_tensor = context->input(context_input_index++); + const Tensor& avg_tensor = context->input(context_input_index++); + const Tensor& std_tensor = context->input(context_input_index++); + // set size of the sample. assume 't' is [[[1, 1, 1], [2, 2, 2]], [[3, 3, 3], [4, 4, 4]]], then shape(t) ==> [2, 2, 3] + OP_REQUIRES (context, (coord_tensor.shape().dims() == 2), errors::InvalidArgument ("Dim of coord should be 2")); + OP_REQUIRES (context, (type_tensor.shape().dims() == 2), errors::InvalidArgument ("Dim of type should be 2")); + OP_REQUIRES (context, (natoms_tensor.shape().dims() == 1), errors::InvalidArgument ("Dim of natoms should be 1")); + OP_REQUIRES (context, (box_tensor.shape().dims() == 2), errors::InvalidArgument ("Dim of box should be 2")); + OP_REQUIRES (context, (mesh_tensor.shape().dims() == 1), errors::InvalidArgument ("Dim of mesh should be 1")); + OP_REQUIRES (context, (avg_tensor.shape().dims() == 2), errors::InvalidArgument ("Dim of avg should be 2")); + OP_REQUIRES (context, (std_tensor.shape().dims() == 2), errors::InvalidArgument ("Dim of std should be 2")); + OP_REQUIRES (context, (sec_r.back() == 0), errors::InvalidArgument ("Rotational free descriptor only support all-angular information: sel_r should be all zero.")); + OP_REQUIRES (context, (natoms_tensor.shape().dim_size(0) >= 3), errors::InvalidArgument ("number of atoms should be larger than (or equal to) 3")); + DeviceFunctor() ( + device, + context->eigen_device() + ); + const int * natoms = natoms_tensor.flat().data(); + int nloc = natoms[0]; + int nall = natoms[1]; + int ntypes = natoms_tensor.shape().dim_size(0) - 2; + int nsamples = coord_tensor.shape().dim_size(0); + //// check the sizes + OP_REQUIRES (context, (nsamples == type_tensor.shape().dim_size(0)), errors::InvalidArgument ("number of samples should match")); + OP_REQUIRES (context, (nsamples == box_tensor.shape().dim_size(0)), errors::InvalidArgument ("number of samples should match")); + OP_REQUIRES (context, (ntypes == avg_tensor.shape().dim_size(0)), errors::InvalidArgument ("number of avg should be ntype")); + OP_REQUIRES (context, (ntypes == std_tensor.shape().dim_size(0)), errors::InvalidArgument ("number of std should be ntype")); + + OP_REQUIRES (context, (nall * 3 == coord_tensor.shape().dim_size(1)), errors::InvalidArgument ("number of atoms should match")); + OP_REQUIRES (context, (nall == type_tensor.shape().dim_size(1)), errors::InvalidArgument ("number of atoms should match")); + OP_REQUIRES (context, (9 == box_tensor.shape().dim_size(1)), errors::InvalidArgument ("number of box should be 9")); + OP_REQUIRES (context, (ndescrpt == avg_tensor.shape().dim_size(1)), errors::InvalidArgument ("number of avg should be ndescrpt")); + OP_REQUIRES (context, (ndescrpt == std_tensor.shape().dim_size(1)), errors::InvalidArgument ("number of std should be ndescrpt")); + + OP_REQUIRES (context, (1 == int(sel_a.size())), errors::InvalidArgument ("the length of sel array should be 1 in this op")); + OP_REQUIRES (context, (1 == int(sel_r.size())), errors::InvalidArgument ("the length of sel array should be 1 in this op")); + + int nei_mode = 0; + bool b_nlist_map = false; + if (mesh_tensor.shape().dim_size(0) == 16) { + // lammps neighbor list + nei_mode = 3; + } + else if (mesh_tensor.shape().dim_size(0) == 6) { + // manual copied pbc + assert (nloc == nall); + nei_mode = 1; + b_nlist_map = true; + } + else if (mesh_tensor.shape().dim_size(0) == 0) { + // no pbc + assert (nloc == nall); + nei_mode = -1; + } + else { + throw deepmd::deepmd_exception("invalid mesh tensor"); + } + + // Create output tensors + TensorShape descrpt_shape ; + descrpt_shape.AddDim (nsamples); + descrpt_shape.AddDim (int_64(nloc) * ndescrpt); + TensorShape descrpt_deriv_shape ; + descrpt_deriv_shape.AddDim (nsamples); + descrpt_deriv_shape.AddDim (int_64(nloc) * ndescrpt * 3); + TensorShape rij_shape ; + rij_shape.AddDim (nsamples); + rij_shape.AddDim (int_64(nloc) * nnei * 3); + TensorShape nlist_shape ; + nlist_shape.AddDim (nsamples); + nlist_shape.AddDim (int_64(nloc) * nnei); + TensorShape ntype_shape ; + ntype_shape.AddDim (nsamples); + ntype_shape.AddDim (nloc * nnei); + TensorShape nmask_shape ; + nmask_shape.AddDim (nsamples); + nmask_shape.AddDim (nloc * nnei); + // define output tensor + int context_output_index = 0; + Tensor* descrpt_tensor = NULL; + Tensor* descrpt_deriv_tensor = NULL; + Tensor* rij_tensor = NULL; + Tensor* nlist_tensor = NULL; + Tensor* ntype_tensor = NULL; + Tensor* nmask_tensor = NULL; + OP_REQUIRES_OK(context, context->allocate_output( + context_output_index++, + descrpt_shape, + &descrpt_tensor)); + OP_REQUIRES_OK(context, context->allocate_output( + context_output_index++, + descrpt_deriv_shape, + &descrpt_deriv_tensor)); + OP_REQUIRES_OK(context, context->allocate_output( + context_output_index++, + rij_shape, + &rij_tensor)); + OP_REQUIRES_OK(context, context->allocate_output( + context_output_index++, + nlist_shape, + &nlist_tensor)); + OP_REQUIRES_OK(context, context->allocate_output( + context_output_index++, + ntype_shape, + &ntype_tensor)); + OP_REQUIRES_OK(context, context->allocate_output( + context_output_index++, + nmask_shape, + &nmask_tensor)); + + FPTYPE * p_em = descrpt_tensor->flat().data(); + FPTYPE * p_em_deriv = descrpt_deriv_tensor->flat().data(); + FPTYPE * p_rij = rij_tensor->flat().data(); + int * p_nlist = nlist_tensor->flat().data(); + int * p_ntype = ntype_tensor->flat().data(); + bool * p_nmask = nmask_tensor->flat().data(); + const FPTYPE * p_coord = coord_tensor.flat().data(); + const FPTYPE * p_box = box_tensor.flat().data(); + const FPTYPE * avg = avg_tensor.flat().data(); + const FPTYPE * std = std_tensor.flat().data(); + const int * p_type = type_tensor.flat().data(); + + // loop over samples + for(int_64 ff = 0; ff < nsamples; ++ff){ + FPTYPE * em = p_em + ff*nloc*ndescrpt; + FPTYPE * em_deriv = p_em_deriv + ff*nloc*ndescrpt*3; + FPTYPE * rij = p_rij + ff*nloc*nnei*3; + int * nlist = p_nlist + ff*nloc*nnei; + int * ntype = p_ntype + ff*nloc*nnei; + bool * nmask = p_nmask + ff*nloc*nnei; + const FPTYPE * coord = p_coord + ff*nall*3; + const FPTYPE * box = p_box + ff*9; + const int * type = p_type + ff*nall; + + if(device == "GPU") { + #if GOOGLE_CUDA + int * idx_mapping = NULL; + int * ilist = NULL, * numneigh = NULL; + int ** firstneigh = NULL; + deepmd::malloc_device_memory(firstneigh, nloc); + int * jlist = NULL; + FPTYPE * coord_cpy; + int * type_cpy; + int frame_nall = nall; + int mesh_tensor_size = static_cast(mesh_tensor.NumElements()); + std::vector tensor_list(7); + Tensor fake_type; // all zeros + TensorShape fake_type_shape; + fake_type_shape.AddDim(nall); + OP_REQUIRES_OK(context, context->allocate_temp(DT_INT32, fake_type_shape, &fake_type)); + deepmd::memset_device_memory(fake_type.flat().data(), 0, nall); + const int * f_type = fake_type.flat().data(); + // prepare coord and nlist + _prepare_coord_nlist_gpu( + context, &tensor_list[0], &coord, coord_cpy, &f_type, type_cpy, idx_mapping, + gpu_inlist, ilist, numneigh, firstneigh, jlist, nbor_list_dev, + frame_nall, mem_cpy, mem_nnei, max_nbor_size, + box, mesh_tensor.flat().data(), mesh_tensor_size, nloc, nei_mode, rcut_r, max_cpy_trial, max_nnei_trial); + + // allocate temp memory, temp memory must not be used after this operation! + Tensor int_temp; + TensorShape int_shape; + int_shape.AddDim(sec_a.size() + int_64(nloc) * sec_a.size() + nloc); + OP_REQUIRES_OK(context, context->allocate_temp(DT_INT32, int_shape, &int_temp)); + Tensor uint64_temp; + TensorShape uint64_shape; + uint64_shape.AddDim(int_64(nloc) * max_nbor_size * 2); + OP_REQUIRES_OK(context, context->allocate_temp(DT_UINT64, uint64_shape, &uint64_temp)); + array_int = int_temp.flat().data(); + array_longlong = uint64_temp.flat().data(); + // launch the gpu(nv) compute function + deepmd::prod_env_mat_a_gpu_cuda( + em, em_deriv, rij, nlist, + coord, type, gpu_inlist, array_int, array_longlong, max_nbor_size, avg, std, nloc, frame_nall, rcut_r, rcut_r_smth, sec_a, f_type); + _map_nei_info_gpu(nlist, ntype, nmask, type, idx_mapping, nloc, nnei, ntypes, b_nlist_map); + deepmd::delete_device_memory(firstneigh); + #endif //GOOGLE_CUDA + + #if TENSORFLOW_USE_ROCM + int * idx_mapping = NULL; + int * ilist = NULL, * numneigh = NULL; + int ** firstneigh = NULL; + deepmd::malloc_device_memory(firstneigh, nloc); + int * jlist = NULL; + FPTYPE * coord_cpy; + int * type_cpy; + int frame_nall = nall; + int mesh_tensor_size = static_cast(mesh_tensor.NumElements()); + std::vector tensor_list(7); + Tensor fake_type; // all zeros + TensorShape fake_type_shape; + fake_type_shape.AddDim(nall); + OP_REQUIRES_OK(context, context->allocate_temp(DT_INT32, fake_type_shape, &fake_type)); + deepmd::memset_device_memory(fake_type.flat().data(), 0, nall); + const int * f_type = fake_type.flat().data(); + // prepare coord and nlist + _prepare_coord_nlist_gpu_rocm( + context, &tensor_list[0], &coord, coord_cpy, &f_type, type_cpy, idx_mapping, + gpu_inlist, ilist, numneigh, firstneigh, jlist, nbor_list_dev, + frame_nall, mem_cpy, mem_nnei, max_nbor_size, + box, mesh_tensor.flat().data(), mesh_tensor_size, nloc, nei_mode, rcut_r, max_cpy_trial, max_nnei_trial); + + // allocate temp memory, temp memory must not be used after this operation! + Tensor int_temp; + TensorShape int_shape; + int_shape.AddDim(sec_a.size() + int_64(nloc) * sec_a.size() + nloc); + OP_REQUIRES_OK(context, context->allocate_temp(DT_INT32, int_shape, &int_temp)); + Tensor uint64_temp; + TensorShape uint64_shape; + uint64_shape.AddDim(int_64(nloc) * max_nbor_size * 2); + OP_REQUIRES_OK(context, context->allocate_temp(DT_UINT64, uint64_shape, &uint64_temp)); + array_int = int_temp.flat().data(); + array_longlong = uint64_temp.flat().data(); + + // launch the gpu(nv) compute function + deepmd::prod_env_mat_a_gpu_rocm( + em, em_deriv, rij, nlist, + coord, type, gpu_inlist, array_int, array_longlong, max_nbor_size, avg, std, nloc, frame_nall, rcut_r, rcut_r_smth, sec_a, f_type); + _map_nei_info_gpu_rocm(nlist, ntype, nmask, type, idx_mapping, nloc, nnei, ntypes, b_nlist_map); + deepmd::delete_device_memory(firstneigh); + #endif //TENSORFLOW_USE_ROCM + } + else if (device == "CPU") { + deepmd::InputNlist inlist; + // some buffers, be freed after the evaluation of this frame + std::vector idx_mapping; + std::vector ilist(nloc), numneigh(nloc); + std::vector firstneigh(nloc); + std::vector> jlist(nloc); + std::vector coord_cpy; + std::vector type_cpy; + int frame_nall = nall; + std::vector fake_type(nall, 0); + const int * f_type = &fake_type[0]; + // prepare coord and nlist + _prepare_coord_nlist_cpu( + context, &coord, coord_cpy, &f_type, type_cpy, idx_mapping, + inlist, ilist, numneigh, firstneigh, jlist, + frame_nall, mem_cpy, mem_nnei, max_nbor_size, + box, mesh_tensor.flat().data(), nloc, nei_mode, rcut_r, max_cpy_trial, max_nnei_trial); + // launch the cpu compute function + deepmd::prod_env_mat_a_cpu( + em, em_deriv, rij, nlist, + coord, type, inlist, max_nbor_size, avg, std, nloc, frame_nall, rcut_r, rcut_r_smth, sec_a, f_type); + // do nlist mapping if coords were copied + _map_nei_info_cpu(nlist, ntype, nmask, type, &idx_mapping[0], nloc, nnei, ntypes, b_nlist_map); + } + } + } + +///////////////////////////////////////////////////////////////////////////////////////////// +private: + float rcut_a; + float rcut_r; + float rcut_r_smth; + std::vector sel_r; + std::vector sel_a; + std::vector sec_a; + std::vector sec_r; + int ndescrpt, ndescrpt_a, ndescrpt_r; + int nnei, nnei_a, nnei_r, nloc, nall, max_nbor_size; + int mem_cpy, max_cpy_trial; + int mem_nnei, max_nnei_trial; + std::string device; + int * array_int = NULL; + unsigned long long * array_longlong = NULL; + deepmd::InputNlist gpu_inlist; + int * nbor_list_dev = NULL; +}; template @@ -989,6 +1362,21 @@ _map_nlist_cpu( } } +static void +_map_nei_info_cpu( + int * nlist, + int * ntype, + bool * nmask, + const int * type, + const int * idx_mapping, + const int & nloc, + const int & nnei, + const int & ntypes, + const bool & b_nlist_map) +{ + deepmd::use_nei_info_cpu(nlist, ntype, nmask, type, idx_mapping, nloc, nnei, ntypes, b_nlist_map); +} + template static void _prepare_coord_nlist_cpu( @@ -1188,6 +1576,21 @@ _map_nlist_gpu( deepmd::use_nlist_map(nlist, idx_mapping, nloc, nnei); } +static void +_map_nei_info_gpu( + int * nlist, + int * ntype, + bool * nmask, + const int * type, + const int * idx_mapping, + const int & nloc, + const int & nnei, + const int & ntypes, + const bool & b_nlist_map) +{ + deepmd::use_nei_info_gpu(nlist, ntype, nmask, type, idx_mapping, nloc, nnei, ntypes, b_nlist_map); +} + template static void _prepare_coord_nlist_gpu( @@ -1403,6 +1806,22 @@ _map_nlist_gpu_rocm( deepmd::use_nlist_map(nlist, idx_mapping, nloc, nnei); } +static void +_map_nei_info_gpu_rocm( + int * nlist, + int * ntype, + bool * nmask, + const int * type, + const int * idx_mapping, + const int & nloc, + const int & nnei, + const int & ntypes, + const bool & b_nlist_map) +{ + deepmd::use_nei_info_gpu_rocm(nlist, ntype, nmask, type, idx_mapping, nloc, nnei, ntypes, b_nlist_map); +} + + template static void _prepare_coord_nlist_gpu_rocm( @@ -1484,6 +1903,9 @@ REGISTER_KERNEL_BUILDER( REGISTER_KERNEL_BUILDER( \ Name("ProdEnvMatR").Device(DEVICE_CPU).TypeConstraint("T"), \ ProdEnvMatROp); \ +REGISTER_KERNEL_BUILDER( \ + Name("ProdEnvMatAMix").Device(DEVICE_CPU).TypeConstraint("T"), \ + ProdEnvMatAMixOp); \ REGISTER_KERNEL_BUILDER( \ Name("DescrptSeA").Device(DEVICE_CPU).TypeConstraint("T"), \ ProdEnvMatAOp); \ @@ -1506,6 +1928,9 @@ REGISTER_KERNEL_BUILDER( REGISTER_KERNEL_BUILDER( \ Name("ProdEnvMatR").Device(DEVICE_GPU).TypeConstraint("T").HostMemory("natoms").HostMemory("box"), \ ProdEnvMatROp); \ +REGISTER_KERNEL_BUILDER( \ + Name("ProdEnvMatAMix").Device(DEVICE_GPU).TypeConstraint("T").HostMemory("natoms").HostMemory("box"), \ + ProdEnvMatAMixOp); \ REGISTER_KERNEL_BUILDER( \ Name("DescrptSeA").Device(DEVICE_GPU).TypeConstraint("T").HostMemory("natoms").HostMemory("box"), \ ProdEnvMatAOp); \ diff --git a/source/tests/common.py b/source/tests/common.py index e767f8a3eb..38c5bc1aee 100644 --- a/source/tests/common.py +++ b/source/tests/common.py @@ -24,9 +24,11 @@ def j_loader(filename): def del_data(): if os.path.isdir('system'): shutil.rmtree('system') + if os.path.isdir('system_mixed_type'): + shutil.rmtree('system_mixed_type') -def gen_data(nframes = 1) : - tmpdata = Data(rand_pert = 0.1, seed = 1, nframes = nframes) +def gen_data_type_specific(nframes = 1): + tmpdata = Data(rand_pert=0.1, seed=1, nframes=nframes) sys = dpdata.LabeledSystem() sys.data['atom_names'] = ['foo', 'bar'] sys.data['coords'] = tmpdata.coord @@ -34,14 +36,41 @@ def gen_data(nframes = 1) : sys.data['cells'] = tmpdata.cell nframes = tmpdata.nframes natoms = tmpdata.natoms - sys.data['coords'] = sys.data['coords'].reshape([nframes,natoms,3]) - sys.data['cells'] = sys.data['cells'].reshape([nframes,3,3]) - sys.data['energies'] = np.zeros([nframes,1]) - sys.data['forces'] = np.zeros([nframes,natoms,3]) - sys.to_deepmd_npy('system', prec=np.float64) + sys.data['coords'] = sys.data['coords'].reshape([nframes, natoms, 3]) + sys.data['cells'] = sys.data['cells'].reshape([nframes, 3, 3]) + sys.data['energies'] = np.zeros([nframes, 1]) + sys.data['forces'] = np.zeros([nframes, natoms, 3]) + sys.to_deepmd_npy('system', prec=np.float64) np.save('system/set.000/fparam.npy', tmpdata.fparam) np.save('system/set.000/aparam.npy', tmpdata.aparam.reshape([nframes, natoms, 2])) +def gen_data_mixed_type(nframes = 1): + tmpdata = Data(rand_pert=0.1, seed=1, nframes=nframes) + sys = dpdata.LabeledSystem() + real_type_map = ['foo', 'bar'] + sys.data['atom_names'] = ['X'] + sys.data['coords'] = tmpdata.coord + sys.data['atom_types'] = np.zeros_like(tmpdata.atype) + sys.data['cells'] = tmpdata.cell + nframes = tmpdata.nframes + natoms = tmpdata.natoms + sys.data['coords'] = sys.data['coords'].reshape([nframes, natoms, 3]) + sys.data['cells'] = sys.data['cells'].reshape([nframes, 3, 3]) + sys.data['energies'] = np.zeros([nframes, 1]) + sys.data['forces'] = np.zeros([nframes, natoms, 3]) + sys.to_deepmd_npy('system_mixed_type', prec=np.float64) + np.savetxt('system_mixed_type/type_map.raw', real_type_map, fmt='%s') + np.save('system_mixed_type/set.000/real_atom_types.npy', tmpdata.atype.reshape(1, -1).repeat(nframes, 0)) + np.save('system_mixed_type/set.000/fparam.npy', tmpdata.fparam) + np.save('system_mixed_type/set.000/aparam.npy', tmpdata.aparam.reshape([nframes, natoms, 2])) + +def gen_data(nframes = 1, mixed_type=False) : + if not mixed_type: + gen_data_type_specific(nframes) + else: + gen_data_mixed_type(nframes) + + class Data(): def __init__ (self, rand_pert = 0.1, @@ -377,4 +406,4 @@ def strerch_box(old_coord, old_box, new_box): obox = old_box.reshape(3,3) nbox = new_box.reshape(3,3) ncoord = ocoord @ np.linalg.inv(obox) @ nbox - return ncoord.reshape(old_coord.shape) \ No newline at end of file + return ncoord.reshape(old_coord.shape) diff --git a/source/tests/test_data_large_batch.py b/source/tests/test_data_large_batch.py new file mode 100644 index 0000000000..f84aaeedf6 --- /dev/null +++ b/source/tests/test_data_large_batch.py @@ -0,0 +1,173 @@ +import dpdata, os, sys, unittest +import numpy as np +from deepmd.env import tf +import pickle +from common import Data, gen_data, j_loader + +from deepmd.utils.data_system import DeepmdDataSystem +from deepmd.descriptor import DescrptSeAtten +from deepmd.common import data_requirement +from deepmd.fit import EnerFitting +from deepmd.model import EnerModel +from deepmd.utils.type_embed import TypeEmbedNet +from deepmd.common import j_must_have +from common import tf +from packaging.version import parse as parse_version + +GLOBAL_ENER_FLOAT_PRECISION = tf.float64 +GLOBAL_TF_FLOAT_PRECISION = tf.float64 +GLOBAL_NP_FLOAT_PRECISION = np.float64 + + +@unittest.skipIf(parse_version(tf.__version__) < parse_version("1.15"), + f"The current tf version {tf.__version__} is too low to run the new testing model.") +class TestDataLargeBatch(tf.test.TestCase): + def setUp(self): + gen_data(mixed_type=True) + + def test_data_mixed_type(self): + jfile = 'water_se_atten_mixed_type.json' + jdata = j_loader(jfile) + + systems = j_must_have(jdata, 'systems') + batch_size = 1 + test_size = 1 + rcut = j_must_have(jdata['model']['descriptor'], 'rcut') + + data = DeepmdDataSystem(systems, batch_size, test_size, rcut) + data_requirement = {'energy': {'ndof': 1, + 'atomic': False, + 'must': False, + 'high_prec': True, + 'type_sel': None, + 'repeat': 1, + 'default': 0.0}, + 'force': {'ndof': 3, + 'atomic': True, + 'must': False, + 'high_prec': False, + 'type_sel': None, + 'repeat': 1, + 'default': 0.0}, + 'virial': {'ndof': 9, + 'atomic': False, + 'must': False, + 'high_prec': False, + 'type_sel': None, + 'repeat': 1, + 'default': 0.0}, + 'atom_ener': {'ndof': 1, + 'atomic': True, + 'must': False, + 'high_prec': False, + 'type_sel': None, + 'repeat': 1, + 'default': 0.0}, + 'atom_pref': {'ndof': 1, + 'atomic': True, + 'must': False, + 'high_prec': False, + 'type_sel': None, + 'repeat': 3, + 'default': 0.0}} + data.add_dict(data_requirement) + + test_data = data.get_test() + numb_test = 1 + + jdata['model']['descriptor'].pop('type', None) + jdata['model']['descriptor']['ntypes'] = 2 + descrpt = DescrptSeAtten(**jdata['model']['descriptor'], uniform_seed=True) + jdata['model']['fitting_net']['descrpt'] = descrpt + fitting = EnerFitting(**jdata['model']['fitting_net'], uniform_seed=True) + typeebd_param = jdata['model']['type_embedding'] + typeebd = TypeEmbedNet( + neuron=typeebd_param['neuron'], + resnet_dt=typeebd_param['resnet_dt'], + activation_function=None, + seed=typeebd_param['seed'], + uniform_seed=True, + padding=True) + model = EnerModel(descrpt, fitting, typeebd) + + # model._compute_dstats([test_data['coord']], [test_data['box']], [test_data['type']], [test_data['natoms_vec']], [test_data['default_mesh']]) + input_data = {'coord': [test_data['coord']], + 'box': [test_data['box']], + 'type': [test_data['type']], + 'natoms_vec': [test_data['natoms_vec']], + 'default_mesh': [test_data['default_mesh']], + 'real_natoms_vec': [test_data['real_natoms_vec']] + } + model._compute_input_stat(input_data, mixed_type=True) + model.descrpt.bias_atom_e = np.array([0., 0.]) + + t_energy = tf.placeholder(GLOBAL_ENER_FLOAT_PRECISION, [None], name='t_energy') + t_force = tf.placeholder(GLOBAL_TF_FLOAT_PRECISION, [None], name='t_force') + t_virial = tf.placeholder(GLOBAL_TF_FLOAT_PRECISION, [None], name='t_virial') + t_atom_ener = tf.placeholder(GLOBAL_TF_FLOAT_PRECISION, [None], name='t_atom_ener') + t_coord = tf.placeholder(GLOBAL_TF_FLOAT_PRECISION, [None], name='i_coord') + t_type = tf.placeholder(tf.int32, [None], name='i_type') + t_natoms = tf.placeholder(tf.int32, [model.ntypes + 2], name='i_natoms') + t_box = tf.placeholder(GLOBAL_TF_FLOAT_PRECISION, [None, 9], name='i_box') + t_mesh = tf.placeholder(tf.int32, [None], name='i_mesh') + is_training = tf.placeholder(tf.bool) + t_fparam = None + inputs_dict = {} + + model_pred \ + = model.build(t_coord, + t_type, + t_natoms, + t_box, + t_mesh, + inputs_dict, + suffix="se_atten", + reuse=False) + + energy = model_pred['energy'] + force = model_pred['force'] + virial = model_pred['virial'] + atom_ener = model_pred['atom_ener'] + + feed_dict_test = {t_energy: np.reshape(test_data['energy'][:numb_test], [-1]), + t_force: np.reshape(test_data['force'][:numb_test, :], [-1]), + t_virial: np.reshape(test_data['virial'][:numb_test, :], [-1]), + t_atom_ener: np.reshape(test_data['atom_ener'][:numb_test, :], [-1]), + t_coord: np.reshape(test_data['coord'][:numb_test, :], [-1]), + t_box: test_data['box'][:numb_test, :], + t_type: np.reshape(test_data['type'][:numb_test, :], [-1]), + t_natoms: test_data['natoms_vec'], + t_mesh: test_data['default_mesh'], + is_training: False} + + + sess = self.test_session().__enter__() + sess.run(tf.global_variables_initializer()) + [e, f, v] = sess.run([energy, force, virial], + feed_dict=feed_dict_test) + # print(sess.run(model.type_embedding)) + # np.savetxt('tmp.out', sess.run(descrpt.dout, feed_dict = feed_dict_test), fmt='%.10e') + # # print(sess.run(model.atype_embed, feed_dict = feed_dict_test)) + # print(sess.run(fitting.inputs, feed_dict = feed_dict_test)) + # print(sess.run(fitting.outs, feed_dict = feed_dict_test)) + # print(sess.run(fitting.atype_embed, feed_dict = feed_dict_test)) + + e = e.reshape([-1]) + f = f.reshape([-1]) + v = v.reshape([-1]) + np.savetxt('e.out', e.reshape([1, -1]), delimiter=',') + np.savetxt('f.out', f.reshape([1, -1]), delimiter=',') + np.savetxt('v.out', v.reshape([1, -1]), delimiter=',') + + refe = [6.12188445792698e+01] + reff = [-2.7590100298321299e-03, -2.7392865283639755e-03, 8.5672424478673337e-05, 7.3154109032780492e-03, 7.6754109031673332e-04, -1.0882393042639207e-03, 9.8633073531477645e-03, 3.6631966083397029e-03, -2.2379079261940034e-04, -4.2393697523149913e-03, 4.9491210390296492e-04, 1.6970049039709007e-04, -8.9021867696626039e-03, -4.7967452269658322e-03, 9.2569990351204447e-04, -1.2781517046160920e-03, 2.6103819527704053e-03, 1.3095727849551296e-04] + refv = [-1.0171833662757776e-02, -6.7981543912862021e-03, 6.1480942994810296e-04, -6.7981543912861942e-03, 3.0092645628232335e-03, 3.8060849919518031e-04, 6.1480942994810383e-04, 3.8060849919518036e-04, -5.6890657188056002e-05] + + refe = np.reshape(refe, [-1]) + reff = np.reshape(reff, [-1]) + refv = np.reshape(refv, [-1]) + + places = 10 + np.testing.assert_almost_equal(e, refe, places) + np.testing.assert_almost_equal(f, reff, places) + np.testing.assert_almost_equal(v, refv, places) diff --git a/source/tests/test_descrpt_se_atten.py b/source/tests/test_descrpt_se_atten.py new file mode 100644 index 0000000000..777ba0e6bc --- /dev/null +++ b/source/tests/test_descrpt_se_atten.py @@ -0,0 +1,252 @@ +import dpdata, os, sys, unittest +import numpy as np +from deepmd.env import tf +import pickle +from common import Data, gen_data, j_loader + +from deepmd.utils.data_system import DataSystem +from deepmd.descriptor import DescrptSeAtten +from deepmd.fit import EnerFitting +from deepmd.model import EnerModel +from deepmd.common import j_must_have +from deepmd.utils.type_embed import embed_atom_type, TypeEmbedNet +from common import tf +from packaging.version import parse as parse_version + +GLOBAL_ENER_FLOAT_PRECISION = tf.float64 +GLOBAL_TF_FLOAT_PRECISION = tf.float64 +GLOBAL_NP_FLOAT_PRECISION = np.float64 + + +@unittest.skipIf(parse_version(tf.__version__) < parse_version("1.15"), + f"The current tf version {tf.__version__} is too low to run the new testing model.") +class TestModel(tf.test.TestCase): + def setUp(self): + gen_data(nframes=2) + + def test_descriptor_two_sides(self): + jfile = 'water_se_atten.json' + jdata = j_loader(jfile) + + systems = j_must_have(jdata, 'systems') + set_pfx = j_must_have(jdata, 'set_prefix') + batch_size = j_must_have(jdata, 'batch_size') + test_size = j_must_have(jdata, 'numb_test') + batch_size = 2 + test_size = 1 + stop_batch = j_must_have(jdata, 'stop_batch') + rcut = j_must_have(jdata['model']['descriptor'], 'rcut') + sel = j_must_have(jdata['model']['descriptor'], 'sel') + ntypes = len(jdata['model']['type_map']) + + data = DataSystem(systems, set_pfx, batch_size, test_size, rcut, run_opt=None) + + test_data = data.get_test() + numb_test = 1 + + # set parameters + jdata['model']['descriptor']['neuron'] = [5, 5, 5] + jdata['model']['descriptor']['axis_neuron'] = 2 + typeebd_param = {'neuron': [5], + 'resnet_dt': False, + 'seed': 1, + } + + # init models + typeebd = TypeEmbedNet( + neuron=typeebd_param['neuron'], + activation_function=None, + resnet_dt=typeebd_param['resnet_dt'], + seed=typeebd_param['seed'], + uniform_seed=True, + padding=True + ) + + jdata['model']['descriptor'].pop('type', None) + jdata['model']['descriptor']['ntypes'] = ntypes + descrpt = DescrptSeAtten(**jdata['model']['descriptor'], uniform_seed=True) + + # model._compute_dstats([test_data['coord']], [test_data['box']], [test_data['type']], [test_data['natoms_vec']], [test_data['default_mesh']]) + input_data = {'coord': [test_data['coord']], + 'box': [test_data['box']], + 'type': [test_data['type']], + 'natoms_vec': [test_data['natoms_vec']], + 'default_mesh': [test_data['default_mesh']] + } + descrpt.bias_atom_e = data.compute_energy_shift() + + t_prop_c = tf.placeholder(tf.float32, [5], name='t_prop_c') + t_energy = tf.placeholder(GLOBAL_ENER_FLOAT_PRECISION, [None], name='t_energy') + t_force = tf.placeholder(GLOBAL_TF_FLOAT_PRECISION, [None], name='t_force') + t_virial = tf.placeholder(GLOBAL_TF_FLOAT_PRECISION, [None], name='t_virial') + t_atom_ener = tf.placeholder(GLOBAL_TF_FLOAT_PRECISION, [None], name='t_atom_ener') + t_coord = tf.placeholder(GLOBAL_TF_FLOAT_PRECISION, [None], name='i_coord') + t_type = tf.placeholder(tf.int32, [None], name='i_type') + t_natoms = tf.placeholder(tf.int32, [ntypes + 2], name='i_natoms') + t_box = tf.placeholder(GLOBAL_TF_FLOAT_PRECISION, [None, 9], name='i_box') + t_mesh = tf.placeholder(tf.int32, [None], name='i_mesh') + is_training = tf.placeholder(tf.bool) + t_fparam = None + + type_embedding = typeebd.build( + ntypes, + suffix="_se_atten_type_des_ebd_2sdies" + ) + + dout \ + = descrpt.build( + t_coord, + t_type, + t_natoms, + t_box, + t_mesh, + {'type_embedding': type_embedding}, + reuse=False, + suffix="_se_atten_type_des_2sides" + ) + + feed_dict_test = {t_prop_c: test_data['prop_c'], + t_energy: test_data['energy'][:numb_test], + t_force: np.reshape(test_data['force'][:numb_test, :], [-1]), + t_virial: np.reshape(test_data['virial'][:numb_test, :], [-1]), + t_atom_ener: np.reshape(test_data['atom_ener'][:numb_test, :], [-1]), + t_coord: np.reshape(test_data['coord'][:numb_test, :], [-1]), + t_box: test_data['box'][:numb_test, :], + t_type: np.reshape(test_data['type'][:numb_test, :], [-1]), + t_natoms: test_data['natoms_vec'], + t_mesh: test_data['default_mesh'], + is_training: False} + + sess = self.test_session().__enter__() + sess.run(tf.global_variables_initializer()) + [model_dout] = sess.run([dout], + feed_dict=feed_dict_test) + model_dout = model_dout.reshape([-1]) + + ref_dout = [1.3503570575883254e-04, -9.3606804794552518e-05, -9.3606804794552518e-05, 6.4931435609575354e-05, -3.4432462227712845e-04, 2.3883309310633266e-04, -2.1612770334269806e-04, 1.4980041766865035e-04, + 5.1902342465554648e-04, -3.5995814159000579e-04, 1.0061650355705337e-04, -7.5148260042556979e-05, -7.5148260042556979e-05, 5.6249549384058458e-05, -2.7820514647114664e-04, 2.0819618461713165e-04, + -1.5698895407951743e-04, 1.1721016363267746e-04, 4.0972585703616773e-04, -3.0650763759131061e-04, 7.5599650998659526e-05, -5.8808888720672558e-05, -5.8808888720672558e-05, 4.5766209906762655e-05, + -2.1712714013251668e-04, 1.6899894453623564e-04, -1.2167120597162636e-04, 9.4648599144861605e-05, 3.2200758382615601e-04, -2.5060486486718734e-04, 1.1293831101452813e-04, -7.9512063028041913e-05, + -7.9512063028041913e-05, 5.5979262682797850e-05, -2.9058515610909440e-04, 2.0457554106366365e-04, -1.8732839505532627e-04, 1.3188376232775540e-04, 4.4448730317793450e-04, -3.1292650304617497e-04, + 1.3015885894252541e-04, -8.8816609587789126e-05, -8.8816609587789126e-05, 6.0613949400496957e-05, -3.2308121544925519e-04, 2.2046786823295058e-04, -2.1781481424814687e-04, 1.4862599684199924e-04, + 4.9955378034266583e-04, -3.4089120488765758e-04, 1.0160496779809329e-04, -7.4538471222199861e-05, -7.4538471222199861e-05, 5.4703671679263269e-05, -2.7394267959121653e-04, 2.0103409637607701e-04, + -1.6657135958432620e-04, 1.2219321453198225e-04, 4.1344754259964935e-04, -3.0339251136512270e-04] + + places = 10 + np.testing.assert_almost_equal(model_dout, ref_dout, places) + + def test_descriptor_one_side(self): + jfile = 'water_se_atten.json' + jdata = j_loader(jfile) + + systems = j_must_have(jdata, 'systems') + set_pfx = j_must_have(jdata, 'set_prefix') + batch_size = j_must_have(jdata, 'batch_size') + test_size = j_must_have(jdata, 'numb_test') + batch_size = 1 + test_size = 1 + stop_batch = j_must_have(jdata, 'stop_batch') + rcut = j_must_have(jdata['model']['descriptor'], 'rcut') + sel = j_must_have(jdata['model']['descriptor'], 'sel') + ntypes = len(jdata['model']['type_map']) + + data = DataSystem(systems, set_pfx, batch_size, test_size, rcut, run_opt=None) + + test_data = data.get_test() + numb_test = 1 + + # set parameters + jdata['model']['descriptor']['neuron'] = [5, 5, 5] + jdata['model']['descriptor']['axis_neuron'] = 2 + jdata['model']['descriptor']['type_one_side'] = True + typeebd_param = {'neuron': [5], + 'resnet_dt': False, + 'seed': 1, + } + + # init models + typeebd = TypeEmbedNet( + neuron=typeebd_param['neuron'], + activation_function=None, + resnet_dt=typeebd_param['resnet_dt'], + seed=typeebd_param['seed'], + uniform_seed=True, + padding=True + ) + + jdata['model']['descriptor'].pop('type', None) + jdata['model']['descriptor']['ntypes'] = ntypes + descrpt = DescrptSeAtten(**jdata['model']['descriptor'], uniform_seed=True) + + # model._compute_dstats([test_data['coord']], [test_data['box']], [test_data['type']], [test_data['natoms_vec']], [test_data['default_mesh']]) + input_data = {'coord': [test_data['coord']], + 'box': [test_data['box']], + 'type': [test_data['type']], + 'natoms_vec': [test_data['natoms_vec']], + 'default_mesh': [test_data['default_mesh']] + } + descrpt.bias_atom_e = data.compute_energy_shift() + + t_prop_c = tf.placeholder(tf.float32, [5], name='t_prop_c') + t_energy = tf.placeholder(GLOBAL_ENER_FLOAT_PRECISION, [None], name='t_energy') + t_force = tf.placeholder(GLOBAL_TF_FLOAT_PRECISION, [None], name='t_force') + t_virial = tf.placeholder(GLOBAL_TF_FLOAT_PRECISION, [None], name='t_virial') + t_atom_ener = tf.placeholder(GLOBAL_TF_FLOAT_PRECISION, [None], name='t_atom_ener') + t_coord = tf.placeholder(GLOBAL_TF_FLOAT_PRECISION, [None], name='i_coord') + t_type = tf.placeholder(tf.int32, [None], name='i_type') + t_natoms = tf.placeholder(tf.int32, [ntypes + 2], name='i_natoms') + t_box = tf.placeholder(GLOBAL_TF_FLOAT_PRECISION, [None, 9], name='i_box') + t_mesh = tf.placeholder(tf.int32, [None], name='i_mesh') + is_training = tf.placeholder(tf.bool) + t_fparam = None + + type_embedding = typeebd.build( + ntypes, + suffix="_se_atten_type_des_ebd_1side" + ) + + dout \ + = descrpt.build( + t_coord, + t_type, + t_natoms, + t_box, + t_mesh, + {'type_embedding': type_embedding}, + reuse=False, + suffix="_se_atten_type_des_1side" + ) + + feed_dict_test = {t_prop_c: test_data['prop_c'], + t_energy: test_data['energy'][:numb_test], + t_force: np.reshape(test_data['force'][:numb_test, :], [-1]), + t_virial: np.reshape(test_data['virial'][:numb_test, :], [-1]), + t_atom_ener: np.reshape(test_data['atom_ener'][:numb_test, :], [-1]), + t_coord: np.reshape(test_data['coord'][:numb_test, :], [-1]), + t_box: test_data['box'][:numb_test, :], + t_type: np.reshape(test_data['type'][:numb_test, :], [-1]), + t_natoms: test_data['natoms_vec'], + t_mesh: test_data['default_mesh'], + is_training: False} + + sess = self.test_session().__enter__() + sess.run(tf.global_variables_initializer()) + [model_dout] = sess.run([dout], + feed_dict=feed_dict_test) + model_dout = model_dout.reshape([-1]) + + ref_dout = [8.9336098555659429e-05, -3.8921422089719007e-05, -3.8921422089719007e-05, 1.6975109833017758e-05, -2.9184951813034413e-04, 1.2724836941382651e-04, -1.8062533253590169e-04, 7.8681048972093648e-05, + 4.2206017420030542e-04, -1.8398310612921889e-04, 6.4996467281506633e-05, -3.0812041327073575e-05, -3.0812041327073575e-05, 1.4663988013438402e-05, -2.3274950984084172e-04, 1.1059587214865573e-04, + -1.3043761448464089e-04, 6.1788865409826698e-05, 3.2900269837104958e-04, -1.5623668424484728e-04, 5.0697927477465942e-05, -2.3511768544350768e-05, -2.3511768544350768e-05, 1.0919808814040025e-05, + -1.8622373494960208e-04, 8.6439275444049409e-05, -1.0326450661269683e-04, 4.7880797898768150e-05, 2.6230208262918372e-04, -1.2172811361250681e-04, 7.8240863239649707e-05, -3.2501260967978116e-05, + -3.2501260967978116e-05, 1.3502267073810926e-05, -2.5360559687597850e-04, 1.0535336854834091e-04, -1.6047265448841568e-04, 6.6660202062744658e-05, 3.6833864909272261e-04, -1.5301457671691837e-04, + 9.1148582997925288e-05, -3.6614945467066073e-05, -3.6614945467066073e-05, 1.4709958908948206e-05, -2.8364168092837332e-04, 1.1394466218003484e-04, -1.8721615730559043e-04, 7.5203967811613109e-05, + 4.1632420070310456e-04, -1.6724364343353009e-04, 6.9506193268190631e-05, -3.0228106532898472e-05, -3.0228106532898472e-05, 1.3156705594652870e-05, -2.3740975974826574e-04, 1.0328972070195332e-04, + -1.4218547815143072e-04, 6.1827596642872941e-05, 3.4031715116440432e-04, -1.4804591640658066e-04] + + places = 10 + np.testing.assert_almost_equal(model_dout, ref_dout, places) + + + + diff --git a/source/tests/test_examples.py b/source/tests/test_examples.py index 9bcbc2b3d7..61e1801345 100644 --- a/source/tests/test_examples.py +++ b/source/tests/test_examples.py @@ -17,6 +17,7 @@ p_examples / "water" / "se_e3" / "input.json", p_examples / "water" / "se_e2_a_tebd" / "input.json", p_examples / "water" / "se_e2_a_mixed_prec" / "input.json", + p_examples / "water" / "se_atten" / "input.json", p_examples / "water" / "dplr" / "train" / "dw.json", p_examples / "water" / "dplr" / "train" / "ener.json", p_examples / "nopbc" / "train" / "input.json", diff --git a/source/tests/test_fitting_ener_type.py b/source/tests/test_fitting_ener_type.py index 1eb1002147..5ccb4797ab 100644 --- a/source/tests/test_fitting_ener_type.py +++ b/source/tests/test_fitting_ener_type.py @@ -76,14 +76,17 @@ def test_fitting(self): 0.0005154794374703516,-0.00019422534512034776,-0.00019422534512034776,7.318151797939947e-05,-0.0013576642997136488,0.0005115548790018505,-0.0010275333676074971,0.00038716440070070385,0.0005376426714609369,-0.00020257810468163985, 0.0004482204892297628,-0.00016887749501640607,-0.00016887749501640607,6.364643102775375e-05,-0.001181345877677835,0.0004452029242063362,-0.0008941636427724908,0.0003369586197174627,0.0004677878512312651,-0.00017625260641095753]) type_embedding = np.array([1.4916816460764615,0.2720153234707013,-2.4385153754181985,-1.8454294510880027,2.874575701113528,1.1225116575801295,0.4204818970813372,-2.3784087249787587,-1.5053748251050598,2.769329403073084]) + atype = np.array([0, 0, 1, 1, 1, 1], dtype=np.int32) dout= dout.reshape([-1,10]) type_embedding = type_embedding.reshape([ntypes,-1]) + atype = atype.reshape([-1]) atom_ener = fitting.build(tf.convert_to_tensor(dout), t_natoms, - {'type_embedding':tf.convert_to_tensor(type_embedding)}, - reuse = False, - suffix = "se_a_type_fit_") + {'type_embedding':tf.convert_to_tensor(type_embedding), + 'atype':tf.convert_to_tensor(atype)}, + reuse=False, + suffix="se_a_type_fit_") feed_dict_test = {t_prop_c: test_data['prop_c'], t_energy: test_data['energy'] [:numb_test], diff --git a/source/tests/test_model_se_atten.py b/source/tests/test_model_se_atten.py new file mode 100644 index 0000000000..d8bce857b0 --- /dev/null +++ b/source/tests/test_model_se_atten.py @@ -0,0 +1,138 @@ +import dpdata, os, sys, unittest +import numpy as np +from deepmd.env import tf +import pickle +from common import Data, gen_data, j_loader + +from deepmd.utils.data_system import DataSystem +from deepmd.descriptor import DescrptSeAtten +from deepmd.fit import EnerFitting +from deepmd.model import EnerModel +from deepmd.utils.type_embed import TypeEmbedNet +from deepmd.common import j_must_have +from common import tf +from packaging.version import parse as parse_version + +GLOBAL_ENER_FLOAT_PRECISION = tf.float64 +GLOBAL_TF_FLOAT_PRECISION = tf.float64 +GLOBAL_NP_FLOAT_PRECISION = np.float64 + + +@unittest.skipIf(parse_version(tf.__version__) < parse_version("1.15"), + f"The current tf version {tf.__version__} is too low to run the new testing model.") +class TestModel(tf.test.TestCase): + def setUp(self): + gen_data() + + def test_model(self): + jfile = 'water_se_atten.json' + jdata = j_loader(jfile) + + systems = j_must_have(jdata, 'systems') + set_pfx = j_must_have(jdata, 'set_prefix') + batch_size = j_must_have(jdata, 'batch_size') + test_size = j_must_have(jdata, 'numb_test') + batch_size = 1 + test_size = 1 + stop_batch = j_must_have(jdata, 'stop_batch') + rcut = j_must_have(jdata['model']['descriptor'], 'rcut') + + data = DataSystem(systems, set_pfx, batch_size, test_size, rcut, run_opt=None) + + test_data = data.get_test() + numb_test = 1 + + jdata['model']['descriptor'].pop('type', None) + jdata['model']['descriptor']['ntypes'] = 2 + descrpt = DescrptSeAtten(**jdata['model']['descriptor'], uniform_seed=True) + jdata['model']['fitting_net']['descrpt'] = descrpt + fitting = EnerFitting(**jdata['model']['fitting_net'], uniform_seed=True) + typeebd_param = jdata['model']['type_embedding'] + typeebd = TypeEmbedNet( + neuron=typeebd_param['neuron'], + activation_function=None, + resnet_dt=typeebd_param['resnet_dt'], + seed=typeebd_param['seed'], + uniform_seed=True, + padding=True) + model = EnerModel(descrpt, fitting, typeebd) + + # model._compute_dstats([test_data['coord']], [test_data['box']], [test_data['type']], [test_data['natoms_vec']], [test_data['default_mesh']]) + input_data = {'coord': [test_data['coord']], + 'box': [test_data['box']], + 'type': [test_data['type']], + 'natoms_vec': [test_data['natoms_vec']], + 'default_mesh': [test_data['default_mesh']] + } + model._compute_input_stat(input_data) + model.descrpt.bias_atom_e = data.compute_energy_shift() + + t_prop_c = tf.placeholder(tf.float32, [5], name='t_prop_c') + t_energy = tf.placeholder(GLOBAL_ENER_FLOAT_PRECISION, [None], name='t_energy') + t_force = tf.placeholder(GLOBAL_TF_FLOAT_PRECISION, [None], name='t_force') + t_virial = tf.placeholder(GLOBAL_TF_FLOAT_PRECISION, [None], name='t_virial') + t_atom_ener = tf.placeholder(GLOBAL_TF_FLOAT_PRECISION, [None], name='t_atom_ener') + t_coord = tf.placeholder(GLOBAL_TF_FLOAT_PRECISION, [None], name='i_coord') + t_type = tf.placeholder(tf.int32, [None], name='i_type') + t_natoms = tf.placeholder(tf.int32, [model.ntypes + 2], name='i_natoms') + t_box = tf.placeholder(GLOBAL_TF_FLOAT_PRECISION, [None, 9], name='i_box') + t_mesh = tf.placeholder(tf.int32, [None], name='i_mesh') + is_training = tf.placeholder(tf.bool) + t_fparam = None + inputs_dict = {} + + model_pred \ + = model.build(t_coord, + t_type, + t_natoms, + t_box, + t_mesh, + inputs_dict, + suffix="se_atten", + reuse=False) + energy = model_pred['energy'] + force = model_pred['force'] + virial = model_pred['virial'] + atom_ener = model_pred['atom_ener'] + + feed_dict_test = {t_prop_c: test_data['prop_c'], + t_energy: test_data['energy'][:numb_test], + t_force: np.reshape(test_data['force'][:numb_test, :], [-1]), + t_virial: np.reshape(test_data['virial'][:numb_test, :], [-1]), + t_atom_ener: np.reshape(test_data['atom_ener'][:numb_test, :], [-1]), + t_coord: np.reshape(test_data['coord'][:numb_test, :], [-1]), + t_box: test_data['box'][:numb_test, :], + t_type: np.reshape(test_data['type'][:numb_test, :], [-1]), + t_natoms: test_data['natoms_vec'], + t_mesh: test_data['default_mesh'], + is_training: False} + sess = self.test_session().__enter__() + sess.run(tf.global_variables_initializer()) + [e, f, v] = sess.run([energy, force, virial], + feed_dict=feed_dict_test) + # print(sess.run(model.type_embedding)) + # np.savetxt('tmp.out', sess.run(descrpt.dout, feed_dict = feed_dict_test), fmt='%.10e') + # # print(sess.run(model.atype_embed, feed_dict = feed_dict_test)) + # print(sess.run(fitting.inputs, feed_dict = feed_dict_test)) + # print(sess.run(fitting.outs, feed_dict = feed_dict_test)) + # print(sess.run(fitting.atype_embed, feed_dict = feed_dict_test)) + + e = e.reshape([-1]) + f = f.reshape([-1]) + v = v.reshape([-1]) + np.savetxt('e.out', e.reshape([1, -1]), delimiter=',') + np.savetxt('f.out', f.reshape([1, -1]), delimiter=',') + np.savetxt('v.out', v.reshape([1, -1]), delimiter=',') + + refe = [6.12188445792698e+01] + reff = [-2.7590100298321299e-03, -2.7392865283639755e-03, 8.5672424478673337e-05, 7.3154109032780492e-03, 7.6754109031673332e-04, -1.0882393042639207e-03, 9.8633073531477645e-03, 3.6631966083397029e-03, -2.2379079261940034e-04, -4.2393697523149913e-03, 4.9491210390296492e-04, 1.6970049039709007e-04, -8.9021867696626039e-03, -4.7967452269658322e-03, 9.2569990351204447e-04, -1.2781517046160920e-03, 2.6103819527704053e-03, 1.3095727849551296e-04] + refv = [-1.0171833662757776e-02, -6.7981543912862021e-03, 6.1480942994810296e-04, -6.7981543912861942e-03, 3.0092645628232335e-03, 3.8060849919518031e-04, 6.1480942994810383e-04, 3.8060849919518036e-04, -5.6890657188056002e-05] + + refe = np.reshape(refe, [-1]) + reff = np.reshape(reff, [-1]) + refv = np.reshape(refv, [-1]) + + places = 10 + np.testing.assert_almost_equal(e, refe, places) + np.testing.assert_almost_equal(f, reff, places) + np.testing.assert_almost_equal(v, refv, places) diff --git a/source/tests/water_se_atten.json b/source/tests/water_se_atten.json new file mode 100644 index 0000000000..1a1e02c11c --- /dev/null +++ b/source/tests/water_se_atten.json @@ -0,0 +1,66 @@ +{ + "_comment": " model parameters", + "model" : { + "type_map": ["O", "H"], + "type_embedding":{ + "neuron": [8], + "resnet_dt": false, + "seed": 1 + }, + "descriptor" :{ + "type": "se_atten", + "sel": 120, + "rcut_smth": 5.80, + "rcut": 6.00, + "neuron": [25, 50, 100], + "resnet_dt": false, + "type_one_side": false, + "axis_neuron": 16, + "seed": 1, + "attn": 128, + "attn_layer": 2, + "attn_dotr": true, + "attn_mask": false + }, + "fitting_net" : { + "neuron": [240, 240, 240], + "resnet_dt": true, + "seed": 1 + } + }, + + + "_comment": " traing controls", + "systems": ["system"], + "set_prefix": "set", + "stop_batch": 1000000, + "batch_size": 1, + "start_lr": 0.005, + "decay_steps": 5000, + "decay_rate": 0.95, + + "start_pref_e": 0.02, + "limit_pref_e": 1, + "start_pref_f": 1000, + "limit_pref_f": 1, + "start_pref_v": 0, + "limit_pref_v": 0, + + "seed": 1, + + "_comment": " display and restart", + "_comment": " frequencies counted in batch", + "disp_file": "lcurve.out", + "disp_freq": 100, + "numb_test": 1, + "save_freq": 1000, + "save_ckpt": "model.ckpt", + "load_ckpt": "model.ckpt", + "disp_training": true, + "time_training": true, + "profiling": false, + "profiling_file": "timeline.json", + + "_comment": "that's all" +} + diff --git a/source/tests/water_se_atten_mixed_type.json b/source/tests/water_se_atten_mixed_type.json new file mode 100644 index 0000000000..e83e7c3f39 --- /dev/null +++ b/source/tests/water_se_atten_mixed_type.json @@ -0,0 +1,66 @@ +{ + "_comment": " model parameters", + "model" : { + "type_map": ["O", "H"], + "type_embedding":{ + "neuron": [8], + "resnet_dt": false, + "seed": 1 + }, + "descriptor" :{ + "type": "se_atten", + "sel": 120, + "rcut_smth": 5.80, + "rcut": 6.00, + "neuron": [25, 50, 100], + "resnet_dt": false, + "type_one_side": false, + "axis_neuron": 16, + "seed": 1, + "attn": 128, + "attn_layer": 2, + "attn_dotr": true, + "attn_mask": false + }, + "fitting_net" : { + "neuron": [240, 240, 240], + "resnet_dt": true, + "seed": 1 + } + }, + + + "_comment": " traing controls", + "systems": ["system_mixed_type"], + "set_prefix": "set", + "stop_batch": 1000000, + "batch_size": 1, + "start_lr": 0.005, + "decay_steps": 5000, + "decay_rate": 0.95, + + "start_pref_e": 0.02, + "limit_pref_e": 1, + "start_pref_f": 1000, + "limit_pref_f": 1, + "start_pref_v": 0, + "limit_pref_v": 0, + + "seed": 1, + + "_comment": " display and restart", + "_comment": " frequencies counted in batch", + "disp_file": "lcurve.out", + "disp_freq": 100, + "numb_test": 1, + "save_freq": 1000, + "save_ckpt": "model.ckpt", + "load_ckpt": "model.ckpt", + "disp_training": true, + "time_training": true, + "profiling": false, + "profiling_file": "timeline.json", + + "_comment": "that's all" +} + From b8fde65df153ca1f3b01b10b2ba1667a17159956 Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Sun, 28 Aug 2022 21:05:44 -0400 Subject: [PATCH 04/21] docs: make warnings more prominent (#1879) Make warnings shown on the web page more prominent. --- doc/conf.py | 1 + doc/development/type-embedding.md | 4 +++- doc/install/easy-install.md | 2 ++ doc/model/train-se-e2-a-tebd.md | 4 +++- doc/train/training.md | 4 ++-- 5 files changed, 11 insertions(+), 4 deletions(-) diff --git a/doc/conf.py b/doc/conf.py index aa08ae93b1..c4ac170be0 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -256,6 +256,7 @@ def setup(app): mathjax_path = 'https://cdnjs.cloudflare.com/ajax/libs/mathjax/3.2.0/es5/tex-mml-chtml.min.js' myst_enable_extensions = [ 'dollarmath', + 'colon_fence', ] # fix emoji issue in pdf latex_engine = "xelatex" diff --git a/doc/development/type-embedding.md b/doc/development/type-embedding.md index 8f3c3309d5..c1136bf94d 100644 --- a/doc/development/type-embedding.md +++ b/doc/development/type-embedding.md @@ -73,4 +73,6 @@ build -> _pass_filter -> _filter -> _filter_lower In `fitting net`, it take the descriptor vector as input, whose dimension is [natoms, $M_1\times M_2$]. Because we need to involve information of centric atom in this step, we need to generate a matrix named as `atype_embed` (of dim [natoms, nchanl]), in which each row is the type embedding vector of the specific centric atom. The input is sorted by type of centric atom, we also know the number of a particular atom type (stored in `natoms[2+i]`), thus we get the type vector of centric atom. In the build phrase of fitting net, it will check whether type embedding exist in `input_dict` and fetch them. After that calling `embed_atom_type` function to lookup embedding vector for type vector of centric atom to obtain `atype_embed`, and concat input with it ([input, atype_embed]). The modified input go through `fitting net` to get predicted energy. -**P.S.: You can't apply compression method while using atom type embedding** +:::{note} +You can't apply compression method while using atom type embedding. +::: diff --git a/doc/install/easy-install.md b/doc/install/easy-install.md index db7beab8db..243c099eb8 100644 --- a/doc/install/easy-install.md +++ b/doc/install/easy-install.md @@ -4,7 +4,9 @@ There are various easy methods to install DeePMD-kit. Choose one that you prefer After your easy installation, DeePMD-kit (`dp`) and LAMMPS (`lmp`) will be available to execute. You can try `dp -h` and `lmp -h` to see the help. `mpirun` is also available considering you may want to train models or run LAMMPS in parallel. +:::{note} Note: The off-line packages and conda packages require the [GNU C Library](https://www.gnu.org/software/libc/) 2.17 or above. The GPU version requires [compatible NVIDIA driver](https://docs.nvidia.com/deploy/cuda-compatibility/index.html#minor-version-compatibility) to be installed in advance. It is possible to force conda to [override detection](https://docs.conda.io/projects/conda/en/latest/user-guide/tasks/manage-virtual.html#overriding-detected-packages) when installation, but these requirements are still necessary during runtime. +::: - [Install off-line packages](#install-off-line-packages) - [Install with conda](#install-with-conda) diff --git a/doc/model/train-se-e2-a-tebd.md b/doc/model/train-se-e2-a-tebd.md index c80127939d..4820b6ee2c 100644 --- a/doc/model/train-se-e2-a-tebd.md +++ b/doc/model/train-se-e2-a-tebd.md @@ -41,4 +41,6 @@ $deepmd_source_dir/examples/water/se_e2_a_tebd/input.json ``` See [here](../development/type-embedding.md) for further explanation of `type embedding`. -**P.S.: You can't apply compression method while using atom type embedding** +:::{note} +You can't apply compression method while using atom type embedding. +::: \ No newline at end of file diff --git a/doc/train/training.md b/doc/train/training.md index 1183e03b81..0c04b8196a 100644 --- a/doc/train/training.md +++ b/doc/train/training.md @@ -58,6 +58,6 @@ plt.show() Checkpoints will be written to files with prefix {ref}`save_ckpt ` every {ref}`save_freq ` training steps. -## Warning +:::{warning} It is warned that the example water data (in folder `examples/water/data`) is of very limited amount, is provided only for testing purpose, and should not be used to train a production model. - +::: From dfed5b678ba6e45af9a900307acbadf00374ecef Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Thu, 1 Sep 2022 23:23:01 -0400 Subject: [PATCH 05/21] support initilize parameters from a fitting with suffix (#1885) Note: commonly there is no suffix for a fitting. Signed-off-by: Jinzhe Zeng --- deepmd/fit/dipole.py | 4 ++-- deepmd/fit/ener.py | 2 +- deepmd/fit/polar.py | 2 +- deepmd/utils/graph.py | 25 +++++++++++++++++++------ 4 files changed, 23 insertions(+), 10 deletions(-) diff --git a/deepmd/fit/dipole.py b/deepmd/fit/dipole.py index 383ea17f1f..2935aa2b06 100644 --- a/deepmd/fit/dipole.py +++ b/deepmd/fit/dipole.py @@ -182,7 +182,7 @@ def init_variables(self, suffix : str suffix to name scope """ - self.fitting_net_variables = get_fitting_net_variables_from_graph_def(graph_def) + self.fitting_net_variables = get_fitting_net_variables_from_graph_def(graph_def, suffix=suffix) def enable_mixed_precision(self, mixed_prec : dict = None) -> None: @@ -195,4 +195,4 @@ def enable_mixed_precision(self, mixed_prec : dict = None) -> None: The mixed precision setting used in the embedding net """ self.mixed_prec = mixed_prec - self.fitting_precision = get_precision(mixed_prec['output_prec']) \ No newline at end of file + self.fitting_precision = get_precision(mixed_prec['output_prec']) diff --git a/deepmd/fit/ener.py b/deepmd/fit/ener.py index 876af78493..64258d88ec 100644 --- a/deepmd/fit/ener.py +++ b/deepmd/fit/ener.py @@ -563,7 +563,7 @@ def init_variables(self, suffix : str suffix to name scope """ - self.fitting_net_variables = get_fitting_net_variables_from_graph_def(graph_def) + self.fitting_net_variables = get_fitting_net_variables_from_graph_def(graph_def, suffix=suffix) if self.numb_fparam > 0: self.fparam_avg = get_tensor_by_name_from_graph(graph, 'fitting_attr%s/t_fparam_avg' % suffix) self.fparam_inv_std = get_tensor_by_name_from_graph(graph, 'fitting_attr%s/t_fparam_istd' % suffix) diff --git a/deepmd/fit/polar.py b/deepmd/fit/polar.py index 3f1b7daa6b..3bb3d9966b 100644 --- a/deepmd/fit/polar.py +++ b/deepmd/fit/polar.py @@ -389,7 +389,7 @@ def init_variables(self, suffix : str suffix to name scope """ - self.fitting_net_variables = get_fitting_net_variables_from_graph_def(graph_def) + self.fitting_net_variables = get_fitting_net_variables_from_graph_def(graph_def, suffix=suffix) def enable_mixed_precision(self, mixed_prec : dict = None) -> None: diff --git a/deepmd/utils/graph.py b/deepmd/utils/graph.py index fafca75f20..10e13730c5 100644 --- a/deepmd/utils/graph.py +++ b/deepmd/utils/graph.py @@ -238,7 +238,7 @@ def get_embedding_net_variables(model_file : str, suffix: str = "") -> Dict: return get_embedding_net_variables_from_graph_def(graph_def, suffix=suffix) -def get_fitting_net_nodes_from_graph_def(graph_def: tf.GraphDef) -> Dict: +def get_fitting_net_nodes_from_graph_def(graph_def: tf.GraphDef, suffix: str = "") -> Dict: """ Get the fitting net nodes with the given tf.GraphDef object @@ -246,13 +246,22 @@ def get_fitting_net_nodes_from_graph_def(graph_def: tf.GraphDef) -> Dict: ---------- graph_def The input tf.GraphDef object + suffix + suffix of the scope Returns ---------- Dict The fitting net nodes within the given tf.GraphDef object """ - fitting_net_nodes = get_pattern_nodes_from_graph_def(graph_def, FITTING_NET_PATTERN) + if suffix != "": + fitting_net_pattern = FITTING_NET_PATTERN\ + .replace('/idt', suffix + '/idt')\ + .replace('/bias', suffix + '/bias')\ + .replace('/matrix', suffix + '/matrix') + else: + fitting_net_pattern = FITTING_NET_PATTERN + fitting_net_nodes = get_pattern_nodes_from_graph_def(graph_def, fitting_net_pattern) for key in fitting_net_nodes.keys(): assert key.find('bias') > 0 or key.find('matrix') > 0 or key.find( 'idt') > 0, "currently, only support weight matrix, bias and idt at the model compression process!" @@ -277,7 +286,7 @@ def get_fitting_net_nodes(model_file : str) -> Dict: return get_fitting_net_nodes_from_graph_def(graph_def) -def get_fitting_net_variables_from_graph_def(graph_def : tf.GraphDef) -> Dict: +def get_fitting_net_variables_from_graph_def(graph_def : tf.GraphDef, suffix: str = "") -> Dict: """ Get the fitting net variables with the given tf.GraphDef object @@ -285,6 +294,8 @@ def get_fitting_net_variables_from_graph_def(graph_def : tf.GraphDef) -> Dict: ---------- graph_def The input tf.GraphDef object + suffix + suffix of the scope Returns ---------- @@ -292,7 +303,7 @@ def get_fitting_net_variables_from_graph_def(graph_def : tf.GraphDef) -> Dict: The fitting net variables within the given tf.GraphDef object """ fitting_net_variables = {} - fitting_net_nodes = get_fitting_net_nodes_from_graph_def(graph_def) + fitting_net_nodes = get_fitting_net_nodes_from_graph_def(graph_def, suffix=suffix) for item in fitting_net_nodes: node = fitting_net_nodes[item] dtype= tf.as_dtype(node.dtype).as_numpy_dtype @@ -304,7 +315,7 @@ def get_fitting_net_variables_from_graph_def(graph_def : tf.GraphDef) -> Dict: fitting_net_variables[item] = np.reshape(tensor_value, tensor_shape) return fitting_net_variables -def get_fitting_net_variables(model_file : str) -> Dict: +def get_fitting_net_variables(model_file : str, suffix: str = "") -> Dict: """ Get the fitting net variables with the given frozen model(model_file) @@ -312,6 +323,8 @@ def get_fitting_net_variables(model_file : str) -> Dict: ---------- model_file The input frozen model path + suffix + suffix of the scope Returns ---------- @@ -319,7 +332,7 @@ def get_fitting_net_variables(model_file : str) -> Dict: The fitting net variables within the given frozen model """ _, graph_def = load_graph_def(model_file) - return get_fitting_net_variables_from_graph_def(graph_def) + return get_fitting_net_variables_from_graph_def(graph_def, suffix=suffix) def get_type_embedding_net_nodes_from_graph_def(graph_def: tf.GraphDef, suffix: str = "") -> Dict: From 3362077426198dbe293a5f539c5a889881c64546 Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Sat, 3 Sep 2022 09:32:46 -0400 Subject: [PATCH 06/21] check/update sel for se_atten (#1886) Signed-off-by: Jinzhe Zeng --- deepmd/entrypoints/main.py | 6 ++ deepmd/entrypoints/neighbor_stat.py | 5 +- deepmd/entrypoints/train.py | 32 +++++----- deepmd/utils/argcheck.py | 5 +- deepmd/utils/neighbor_stat.py | 22 +++++-- source/tests/test_train.py | 95 ++++++++++++++++++++++++++++- 6 files changed, 143 insertions(+), 22 deletions(-) diff --git a/deepmd/entrypoints/main.py b/deepmd/entrypoints/main.py index 5546ca15cd..657dff2bbb 100644 --- a/deepmd/entrypoints/main.py +++ b/deepmd/entrypoints/main.py @@ -445,6 +445,12 @@ def main_parser() -> argparse.ArgumentParser: required=True, help="type map", ) + parser_neighbor_stat.add_argument( + "--one-type", + action="store_true", + default=False, + help="treat all types as a single type. Used with se_atten descriptor.", + ) # --version parser.add_argument('--version', action='version', version='DeePMD-kit v%s' % __version__) diff --git a/deepmd/entrypoints/neighbor_stat.py b/deepmd/entrypoints/neighbor_stat.py index eaf8b3efe3..bc3c6430ff 100644 --- a/deepmd/entrypoints/neighbor_stat.py +++ b/deepmd/entrypoints/neighbor_stat.py @@ -12,6 +12,7 @@ def neighbor_stat( system: str, rcut: float, type_map: List[str], + one_type: bool = False, **kwargs, ): """Calculate neighbor statistics. @@ -24,6 +25,8 @@ def neighbor_stat( cutoff radius type_map : list[str] type map + one_type : bool, optional, default=False + treat all types as a single type Examples -------- @@ -42,7 +45,7 @@ def neighbor_stat( type_map=type_map, ) data.get_batch() - nei = NeighborStat(data.get_ntypes(), rcut) + nei = NeighborStat(data.get_ntypes(), rcut, one_type=one_type) min_nbor_dist, max_nbor_size = nei.get_stat(data) log.info("min_nbor_dist: %f" % min_nbor_dist) log.info("max_nbor_size: %s" % str(max_nbor_size)) diff --git a/deepmd/entrypoints/train.py b/deepmd/entrypoints/train.py index 7cdb0307d4..7ffdc7c8b0 100755 --- a/deepmd/entrypoints/train.py +++ b/deepmd/entrypoints/train.py @@ -90,10 +90,7 @@ def train( jdata = normalize(jdata) - if jdata['model']['descriptor']['type'] in ['se_atten'] and isinstance(jdata['model']['descriptor']['sel'], list): - jdata['model']['descriptor']['sel'] = sum(jdata['model']['descriptor']['sel']) - - if not is_compress and not skip_neighbor_stat and jdata['model']['descriptor']['type'] not in ['se_atten']: + if not is_compress and not skip_neighbor_stat: jdata = update_sel(jdata) with open(output, "w") as fp: @@ -253,7 +250,7 @@ def get_type_map(jdata): return jdata['model'].get('type_map', None) -def get_nbor_stat(jdata, rcut): +def get_nbor_stat(jdata, rcut, one_type: bool = False): max_rcut = get_rcut(jdata) type_map = get_type_map(jdata) @@ -268,7 +265,7 @@ def get_nbor_stat(jdata, rcut): map_ntypes = data_ntypes ntypes = max([map_ntypes, data_ntypes]) - neistat = NeighborStat(ntypes, rcut) + neistat = NeighborStat(ntypes, rcut, one_type=one_type) min_nbor_dist, max_nbor_size = neistat.get_stat(train_data) @@ -283,8 +280,8 @@ def get_nbor_stat(jdata, rcut): dtype = tf.int32) return min_nbor_dist, max_nbor_size -def get_sel(jdata, rcut): - _, max_nbor_size = get_nbor_stat(jdata, rcut) +def get_sel(jdata, rcut, one_type: bool = False): + _, max_nbor_size = get_nbor_stat(jdata, rcut, one_type=one_type) return max_nbor_size def get_min_nbor_dist(jdata, rcut): @@ -320,14 +317,20 @@ def wrap_up_4(xx): def update_one_sel(jdata, descriptor): + if descriptor['type'] == 'loc_frame': + return descriptor rcut = descriptor['rcut'] - tmp_sel = get_sel(jdata, rcut) + tmp_sel = get_sel(jdata, rcut, one_type=descriptor['type'] in ('se_atten',)) + sel = descriptor['sel'] + if isinstance(sel, int): + # convert to list and finnally convert back to int + sel = [sel] if parse_auto_sel(descriptor['sel']) : ratio = parse_auto_sel_ratio(descriptor['sel']) - descriptor['sel'] = [int(wrap_up_4(ii * ratio)) for ii in tmp_sel] + descriptor['sel'] = sel = [int(wrap_up_4(ii * ratio)) for ii in tmp_sel] else: # sel is set by user - for ii, (tt, dd) in enumerate(zip(tmp_sel, descriptor['sel'])): + for ii, (tt, dd) in enumerate(zip(tmp_sel, sel)): if dd and tt > dd: # we may skip warning for sel=0, where the user is likely # to exclude such type in the descriptor @@ -336,6 +339,8 @@ def update_one_sel(jdata, descriptor): "not less than %d, but you set it to %d. The accuracy" " of your model may get worse." %(ii, tt, dd) ) + if descriptor['type'] in ('se_atten',): + descriptor['sel'] = sel = sum(sel) return descriptor @@ -344,9 +349,8 @@ def update_sel(jdata): descrpt_data = jdata['model']['descriptor'] if descrpt_data['type'] == 'hybrid': for ii in range(len(descrpt_data['list'])): - if descrpt_data['list'][ii]['type'] != 'loc_frame': - descrpt_data['list'][ii] = update_one_sel(jdata, descrpt_data['list'][ii]) - elif descrpt_data['type'] != 'loc_frame': + descrpt_data['list'][ii] = update_one_sel(jdata, descrpt_data['list'][ii]) + else: descrpt_data = update_one_sel(jdata, descrpt_data) jdata['model']['descriptor'] = descrpt_data return jdata diff --git a/deepmd/utils/argcheck.py b/deepmd/utils/argcheck.py index 7ba62f9953..1f84b7fe60 100644 --- a/deepmd/utils/argcheck.py +++ b/deepmd/utils/argcheck.py @@ -243,7 +243,8 @@ def descrpt_hybrid_args(): def descrpt_se_atten_args(): doc_sel = 'This parameter set the number of selected neighbors. Note that this parameter is a little different from that in other descriptors. Instead of separating each type of atoms, only the summation matters. And this number is highly related with the efficiency, thus one should not make it too large. Usually 200 or less is enough, far away from the GPU limitation 4096. It can be:\n\n\ - `int`. The maximum number of neighbor atoms to be considered. We recommend it to be less than 200. \n\n\ - - `List[int]`. The length of the list should be the same as the number of atom types in the system. `sel[i]` gives the selected number of type-i neighbors. Only the summation of `sel[i]` matters, and it is recommended to be less than 200.' + - `List[int]`. The length of the list should be the same as the number of atom types in the system. `sel[i]` gives the selected number of type-i neighbors. Only the summation of `sel[i]` matters, and it is recommended to be less than 200.\ + - `str`. Can be "auto:factor" or "auto". "factor" is a float number larger than 1. This option will automatically determine the `sel`. In detail it counts the maximal number of neighbors with in the cutoff radius for each type of neighbor, then multiply the maximum by the "factor". Finally the number is wraped up to 4 divisible. The option "auto" is equivalent to "auto:1.1".' doc_rcut = 'The cut-off radius.' doc_rcut_smth = 'Where to start smoothing. For example the 1/r term is smoothed from `rcut` to `rcut_smth`' doc_neuron = 'Number of neurons in each hidden layers of the embedding net. When two layers are of the same size or one layer is twice as large as the previous layer, a skip connection is built.' @@ -262,7 +263,7 @@ def descrpt_se_atten_args(): doc_attn_mask = 'Whether to do mask on the diagonal in the attention matrix' return [ - Argument("sel", [int, list], optional=True, default=200, doc=doc_sel), + Argument("sel", [int, list, str], optional=True, default="auto", doc=doc_sel), Argument("rcut", float, optional=True, default=6.0, doc=doc_rcut), Argument("rcut_smth", float, optional=True, default=0.5, doc=doc_rcut_smth), Argument("neuron", list, optional=True, default=[10, 20, 40], doc=doc_neuron), diff --git a/deepmd/utils/neighbor_stat.py b/deepmd/utils/neighbor_stat.py index 9088cfc8a4..87e5b83774 100644 --- a/deepmd/utils/neighbor_stat.py +++ b/deepmd/utils/neighbor_stat.py @@ -23,15 +23,20 @@ class NeighborStat(): The num of atom types rcut The cut-off radius + one_type : bool, optional, default=False + Treat all types as a single type. """ def __init__(self, ntypes : int, - rcut: float) -> None: + rcut: float, + one_type : bool = False, + ) -> None: """ Constructor """ self.rcut = rcut self.ntypes = ntypes + self.one_type = one_type sub_graph = tf.Graph() def builder(): @@ -41,10 +46,17 @@ def builder(): place_holders['type'] = tf.placeholder(tf.int32, [None, None], name='t_type') place_holders['natoms_vec'] = tf.placeholder(tf.int32, [self.ntypes+2], name='t_natoms') place_holders['default_mesh'] = tf.placeholder(tf.int32, [None], name='t_mesh') + t_type = place_holders['type'] + t_natoms = place_holders['natoms_vec'] + if self.one_type: + # all types = 0, natoms_vec = [natoms, natoms, natoms] + t_type = tf.zeros_like(t_type, dtype=tf.int32) + t_natoms = tf.repeat(t_natoms[0], 3) + _max_nbor_size, _min_nbor_dist \ = op_module.neighbor_stat(place_holders['coord'], - place_holders['type'], - place_holders['natoms_vec'], + t_type, + t_natoms, place_holders['box'], place_holders['default_mesh'], rcut = self.rcut) @@ -74,7 +86,9 @@ def get_stat(self, A list with ntypes integers, denotes the actual achieved max sel """ self.min_nbor_dist = 100.0 - self.max_nbor_size = [0] * self.ntypes + self.max_nbor_size = [0] + if not self.one_type: + self.max_nbor_size *= self.ntypes def feed(): for ii in range(len(data.system_dirs)): diff --git a/source/tests/test_train.py b/source/tests/test_train.py index 2b7fd16d18..ade7ab7cbd 100644 --- a/source/tests/test_train.py +++ b/source/tests/test_train.py @@ -29,6 +29,7 @@ def test_update_one_sel(self, sel_mock): sel_mock.return_value = [10,20] jdata = {} descriptor = { + 'type': 'se_e2_a', 'rcut': 6, 'sel': "auto" } @@ -36,6 +37,7 @@ def test_update_one_sel(self, sel_mock): # self.assertEqual(descriptor['sel'], [11,22]) self.assertEqual(descriptor['sel'], [12,24]) descriptor = { + 'type': 'se_e2_a', 'rcut': 6, 'sel': "auto:1.5" } @@ -113,7 +115,98 @@ def test_update_sel(self, sel_mock): jdata = update_sel(jdata) self.assertEqual(jdata, expected_out) - + @patch("deepmd.entrypoints.train.get_sel") + def test_update_sel_atten_auto(self, sel_mock): + sel_mock.return_value = [25] + jdata = { + 'model' : { + 'descriptor': { + 'type' : 'se_atten', + 'sel' : "auto", + 'rcut': 6, + } + } + } + expected_out = { + 'model' : { + 'descriptor': { + 'type' : 'se_atten', + 'sel' : 28, + 'rcut': 6, + } + } + } + jdata = update_sel(jdata) + self.assertEqual(jdata, expected_out) + + @patch("deepmd.entrypoints.train.get_sel") + def test_update_sel_atten_int(self, sel_mock): + sel_mock.return_value = [25] + jdata = { + 'model' : { + 'descriptor': { + 'type' : 'se_atten', + 'sel' : 30, + 'rcut': 6, + } + } + } + expected_out = { + 'model' : { + 'descriptor': { + 'type' : 'se_atten', + 'sel' : 30, + 'rcut': 6, + } + } + } + jdata = update_sel(jdata) + self.assertEqual(jdata, expected_out) + + @patch("deepmd.entrypoints.train.get_sel") + def test_update_sel_atten_list(self, sel_mock): + sel_mock.return_value = [25] + jdata = { + 'model' : { + 'descriptor': { + 'type' : 'se_atten', + 'sel' : 30, + 'rcut': 6, + } + } + } + expected_out = { + 'model' : { + 'descriptor': { + 'type' : 'se_atten', + 'sel' : 30, + 'rcut': 6, + } + } + } + jdata = update_sel(jdata) + self.assertEqual(jdata, expected_out) + + def test_skip_loc_frame(self): + jdata = { + 'model' : { + 'descriptor': { + 'type' : 'loc_frame', + 'rcut': 6, + } + } + } + expected_out = { + 'model' : { + 'descriptor': { + 'type' : 'loc_frame', + 'rcut': 6, + } + } + } + jdata = update_sel(jdata) + self.assertEqual(jdata, expected_out) + def test_wrap_up_4(self): self.assertEqual(wrap_up_4(12), 3 * 4) self.assertEqual(wrap_up_4(13), 4 * 4) From 0a15d19e97b4ea0216ccaa957cdf3a0ad26f1d4c Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Sat, 3 Sep 2022 09:34:08 -0400 Subject: [PATCH 07/21] add examples to cli help (#1887) ![image](https://user-images.githubusercontent.com/9496702/188026428-f9935050-5874-45ac-8d79-58e54bae6eba.png) Signed-off-by: Jinzhe Zeng --- deepmd/entrypoints/main.py | 50 ++++++++++++++++++++++++++++++++++---- 1 file changed, 45 insertions(+), 5 deletions(-) diff --git a/deepmd/entrypoints/main.py b/deepmd/entrypoints/main.py index 657dff2bbb..045441374a 100644 --- a/deepmd/entrypoints/main.py +++ b/deepmd/entrypoints/main.py @@ -2,6 +2,7 @@ import argparse import logging +import textwrap from pathlib import Path from typing import Dict, List, Optional @@ -45,6 +46,11 @@ def get_ll(log_level: str) -> int: return int_level +class RawTextArgumentDefaultsHelpFormatter( + argparse.RawTextHelpFormatter, argparse.ArgumentDefaultsHelpFormatter +): + """This formatter is used to print multile-line help message with default value.""" + def main_parser() -> argparse.ArgumentParser: """DeePMD-Kit commandline options argument parser. @@ -139,7 +145,13 @@ def main_parser() -> argparse.ArgumentParser: "train", parents=[parser_log, parser_mpi_log], help="train a model", - formatter_class=argparse.ArgumentDefaultsHelpFormatter, + formatter_class=RawTextArgumentDefaultsHelpFormatter, + epilog=textwrap.dedent("""\ + examples: + dp train input.json + dp train input.json --restart model.ckpt + dp train input.json --init-model model.ckpt + """), ) parser_train.add_argument( "INPUT", help="the input parameter file in json or yaml format" @@ -183,7 +195,12 @@ def main_parser() -> argparse.ArgumentParser: "freeze", parents=[parser_log], help="freeze the model", - formatter_class=argparse.ArgumentDefaultsHelpFormatter, + formatter_class=RawTextArgumentDefaultsHelpFormatter, + epilog=textwrap.dedent("""\ + examples: + dp freeze + dp freeze -o graph.pb + """), ) parser_frz.add_argument( "-c", @@ -219,7 +236,11 @@ def main_parser() -> argparse.ArgumentParser: "test", parents=[parser_log], help="test the model", - formatter_class=argparse.ArgumentDefaultsHelpFormatter, + formatter_class=RawTextArgumentDefaultsHelpFormatter, + epilog=textwrap.dedent("""\ + examples: + dp test -m graph.pb -s /path/to/system -n 30 + """), ) parser_tst.add_argument( "-m", @@ -274,7 +295,12 @@ def main_parser() -> argparse.ArgumentParser: "compress", parents=[parser_log, parser_mpi_log], help="compress a model", - formatter_class=argparse.ArgumentDefaultsHelpFormatter, + formatter_class=RawTextArgumentDefaultsHelpFormatter, + epilog=textwrap.dedent("""\ + examples: + dp compress + dp compress -i graph.pb -o compressed.pb + """), ) parser_compress.add_argument( "-i", @@ -355,7 +381,11 @@ def main_parser() -> argparse.ArgumentParser: "model-devi", parents=[parser_log], help="calculate model deviation", - formatter_class=argparse.ArgumentDefaultsHelpFormatter, + formatter_class=RawTextArgumentDefaultsHelpFormatter, + epilog=textwrap.dedent("""\ + examples: + dp model-devi -m graph.000.pb graph.001.pb graph.002.pb graph.003.pb -s ./data -o model_devi.out + """), ) parser_model_devi.add_argument( "-m", @@ -395,6 +425,11 @@ def main_parser() -> argparse.ArgumentParser: 'convert-from', parents=[parser_log], help='convert lower model version to supported version', + formatter_class=RawTextArgumentDefaultsHelpFormatter, + epilog=textwrap.dedent("""\ + examples: + dp convert-from 1.0 -i graph.pb -o graph_new.pb + """), ) parser_transform.add_argument( 'FROM', @@ -422,6 +457,11 @@ def main_parser() -> argparse.ArgumentParser: 'neighbor-stat', parents=[parser_log], help='Calculate neighbor statistics', + formatter_class=RawTextArgumentDefaultsHelpFormatter, + epilog=textwrap.dedent("""\ + examples: + dp neighbor-stat -s data -r 6.0 -t O H + """), ) parser_neighbor_stat.add_argument( "-s", From d0b4a1de16676e5f96290b7a6bf5162bcbb32359 Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Tue, 6 Sep 2022 22:04:14 -0400 Subject: [PATCH 08/21] fix DeprecationWarning of imp (#1896) > deepmd/env.py:8: DeprecationWarning: the imp module is deprecated in favour of importlib and slated for removal in Python 3.12; see the module's documentation for alternative uses See also https://docs.python.org/3/library/imp.html#imp.reload Signed-off-by: Jinzhe Zeng --- deepmd/env.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deepmd/env.py b/deepmd/env.py index a864787411..f0d6e1da4e 100644 --- a/deepmd/env.py +++ b/deepmd/env.py @@ -5,7 +5,7 @@ import re import platform from configparser import ConfigParser -from imp import reload +from importlib import reload from pathlib import Path from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple from packaging.version import Version From bf1a7ccb75a4e2ce89b89e5eddb7fc6d54448e67 Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Tue, 6 Sep 2022 22:04:36 -0400 Subject: [PATCH 09/21] remove `get_platform` in setup.py (#1897) It is not used anymore. Signed-off-by: Jinzhe Zeng --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index bebf8130d5..edc7e0b97b 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,6 @@ import os import site -from distutils.util import get_platform from importlib.machinery import FileFinder from importlib.util import find_spec from pathlib import Path From ff0cc61afbec944439392c7522d53f63192913ad Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Tue, 6 Sep 2022 23:54:55 -0400 Subject: [PATCH 10/21] bump to C++ 17 for TF 2.10 (#1898) Also, (1) replace `GREATER_EQUAL` by `VERSION_GREATER_EQUAL`. `GREATER_EQUAL` does not work with `2.10`; (2) upgrade manylinux to 2_28 to have a newer compiler that supports C++ 17. Signed-off-by: Jinzhe Zeng --- .github/workflows/build_wheel.yml | 2 +- source/CMakeLists.txt | 6 ++++-- source/api_cc/tests/CMakeLists.txt | 4 +++- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build_wheel.yml b/.github/workflows/build_wheel.yml index 873a25ae71..6b749b64fd 100644 --- a/.github/workflows/build_wheel.yml +++ b/.github/workflows/build_wheel.yml @@ -28,7 +28,7 @@ jobs: - name: Build wheels env: CIBW_BUILD: "cp36-* cp37-* cp38-* cp39-* cp310-*" - CIBW_MANYLINUX_X86_64_IMAGE: ghcr.io/deepmodeling/manylinux_2_24_x86_64_tensorflow + CIBW_MANYLINUX_X86_64_IMAGE: ghcr.io/deepmodeling/manylinux_2_28_x86_64_tensorflow CIBW_BEFORE_BUILD: pip install tensorflow CIBW_SKIP: "*-win32 *-manylinux_i686 *-musllinux*" run: | diff --git a/source/CMakeLists.txt b/source/CMakeLists.txt index 9f276c39a0..943d1a14d6 100644 --- a/source/CMakeLists.txt +++ b/source/CMakeLists.txt @@ -87,7 +87,9 @@ endif(USE_TF_PYTHON_LIBS) # find tensorflow, I need tf abi info find_package(tensorflow REQUIRED) -if (TENSORFLOW_VERSION GREATER_EQUAL 2.7) +if (TENSORFLOW_VERSION VERSION_GREATER_EQUAL 2.10) + set (CMAKE_CXX_STANDARD 17) +elseif (TENSORFLOW_VERSION VERSION_GREATER_EQUAL 2.7) set (CMAKE_CXX_STANDARD 14) else() set (CMAKE_CXX_STANDARD 11) @@ -98,7 +100,7 @@ if (MSVC) add_compile_options(/W0 /Zc:__cplusplus /D_USE_MATH_DEFINES /d2ReducedOptimizeHugeFunctions) set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS TRUE) endif() -if (TENSORFLOW_VERSION GREATER_EQUAL 2.4 AND MSVC) +if (TENSORFLOW_VERSION VERSION_GREATER_EQUAL 2.4 AND MSVC) # see TF 2.4 release notes add_compile_options(/Zc:preprocessor) endif() diff --git a/source/api_cc/tests/CMakeLists.txt b/source/api_cc/tests/CMakeLists.txt index 3cde83edbe..076bfe6995 100644 --- a/source/api_cc/tests/CMakeLists.txt +++ b/source/api_cc/tests/CMakeLists.txt @@ -60,7 +60,9 @@ set(apiname "deepmd_api") set(opname "deepmd_op") find_package(tensorflow REQUIRED) -if (TENSORFLOW_VERSION GREATER_EQUAL 2.7) +if (TENSORFLOW_VERSION VERSION_GREATER_EQUAL 2.10) + set (CMAKE_CXX_STANDARD 17) +elseif (TENSORFLOW_VERSION VERSION_GREATER_EQUAL 2.7) set (CMAKE_CXX_STANDARD 14) else() set (CMAKE_CXX_STANDARD 11) From 5b78341e3b9b60f4a0131fb7f039ee2ec55085a1 Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Fri, 9 Sep 2022 05:53:17 -0400 Subject: [PATCH 11/21] compare converted lr with almost equal (#1901) Signed-off-by: Jinzhe Zeng --- source/tests/test_compat_input.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/source/tests/test_compat_input.py b/source/tests/test_compat_input.py index 1933725ac7..cec42e622d 100644 --- a/source/tests/test_compat_input.py +++ b/source/tests/test_compat_input.py @@ -22,7 +22,15 @@ def test_convert_v1_v2(self): jdata0 = j_loader(os.path.join('compat_inputs', 'water_v1.json')) jdata1 = j_loader(os.path.join('compat_inputs', 'water_v2.json')) jdata = convert_input_v1_v2(jdata0, warning = False, dump = None) - self.assertEqual(jdata, jdata1) + self.assertDictAlmostEqual(jdata, jdata1) + + def assertDictAlmostEqual(self, d1, d2, msg=None, places=7): + self.assertEqual(d1.keys(), d2.keys()) + for kk, vv in d1.items(): + if isinstance(vv, dict): + self.assertDictAlmostEqual(d1[kk], d2[kk], msg=msg) + else: + self.assertAlmostEqual(d1[kk], d2[kk], places=places, msg=msg) def test_json_yaml_equal(self): From 3af9d3a840d3064694adbb1d039819b5ea6ee8c3 Mon Sep 17 00:00:00 2001 From: Duo <50307526+iProzd@users.noreply.github.com> Date: Fri, 9 Sep 2022 17:53:55 +0800 Subject: [PATCH 12/21] Fix bugs when init_frz_model using tebd. (#1891) Fix bugs when init_frz_model using tebd. Add init_variables for attention variables. Fix precision of self.t_bias_atom_e in se_atten.py. Change the default values in argcheck for se_atten (typos). Improve the name scope in se_atten.py. Delete some redundant variables in se_atten.py. Move tf.constant from se_a.__init__() into build. Add UTs for init_frz_model. Co-authored-by: Jinzhe Zeng --- deepmd/descriptor/se_a.py | 10 +- deepmd/descriptor/se_atten.py | 86 ++++++--- deepmd/env.py | 25 +++ deepmd/fit/ener.py | 17 +- deepmd/train/trainer.py | 9 +- deepmd/utils/argcheck.py | 6 +- deepmd/utils/graph.py | 62 ++++++- deepmd/utils/network.py | 9 +- .../tests/init_frz_model/data/set.000/box.npy | Bin 0 -> 200 bytes .../init_frz_model/data/set.000/coord.npy | Bin 0 -> 272 bytes .../init_frz_model/data/set.000/energy.npy | Bin 0 -> 136 bytes .../init_frz_model/data/set.000/force.npy | Bin 0 -> 272 bytes source/tests/init_frz_model/data/type.raw | 6 + source/tests/init_frz_model/data/type_map.raw | 2 + source/tests/init_frz_model/input.json | 66 +++++++ source/tests/test_init_frz_model_se_a.py | 161 +++++++++++++++++ source/tests/test_init_frz_model_se_a_type.py | 165 +++++++++++++++++ source/tests/test_init_frz_model_se_atten.py | 167 ++++++++++++++++++ source/tests/test_init_frz_model_se_r.py | 167 ++++++++++++++++++ 19 files changed, 908 insertions(+), 50 deletions(-) create mode 100644 source/tests/init_frz_model/data/set.000/box.npy create mode 100644 source/tests/init_frz_model/data/set.000/coord.npy create mode 100644 source/tests/init_frz_model/data/set.000/energy.npy create mode 100644 source/tests/init_frz_model/data/set.000/force.npy create mode 100644 source/tests/init_frz_model/data/type.raw create mode 100644 source/tests/init_frz_model/data/type_map.raw create mode 100644 source/tests/init_frz_model/input.json create mode 100644 source/tests/test_init_frz_model_se_a.py create mode 100644 source/tests/test_init_frz_model_se_a_type.py create mode 100644 source/tests/test_init_frz_model_se_atten.py create mode 100644 source/tests/test_init_frz_model_se_r.py diff --git a/deepmd/descriptor/se_a.py b/deepmd/descriptor/se_a.py index b568d8be71..9b54daaa94 100644 --- a/deepmd/descriptor/se_a.py +++ b/deepmd/descriptor/se_a.py @@ -164,10 +164,7 @@ def __init__ (self, self.embedding_net_variables = None self.mixed_prec = None self.place_holders = {} - nei_type = np.array([]) - for ii in range(self.ntypes): - nei_type = np.append(nei_type, ii * np.ones(self.sel_a[ii])) # like a mask - self.nei_type = tf.constant(nei_type, dtype = tf.int32) + self.nei_type = np.repeat(np.arange(self.ntypes), self.sel_a) # like a mask avg_zero = np.zeros([self.ntypes,self.ndescrpt]).astype(GLOBAL_NP_FLOAT_PRECISION) std_ones = np.ones ([self.ntypes,self.ndescrpt]).astype(GLOBAL_NP_FLOAT_PRECISION) @@ -673,8 +670,9 @@ def _concat_type_embedding( embedding: environment of each atom represented by embedding. ''' - te_out_dim = type_embedding.get_shape().as_list()[-1] - nei_embed = tf.nn.embedding_lookup(type_embedding,tf.cast(self.nei_type,dtype=tf.int32)) # shape is [self.nnei, 1+te_out_dim] + te_out_dim = type_embedding.get_shape().as_list()[-1] + self.t_nei_type = tf.constant(self.nei_type, dtype=tf.int32) + nei_embed = tf.nn.embedding_lookup(type_embedding,tf.cast(self.t_nei_type,dtype=tf.int32)) # shape is [self.nnei, 1+te_out_dim] nei_embed = tf.tile(nei_embed,(nframes*natoms[0],1)) # shape is [nframes*natoms[0]*self.nnei, te_out_dim] nei_embed = tf.reshape(nei_embed,[-1,te_out_dim]) embedding_input = tf.concat([xyz_scatter,nei_embed],1) # shape is [nframes*natoms[0]*self.nnei, 1+te_out_dim] diff --git a/deepmd/descriptor/se_atten.py b/deepmd/descriptor/se_atten.py index ead4f93d46..ee2ab39a7f 100644 --- a/deepmd/descriptor/se_atten.py +++ b/deepmd/descriptor/se_atten.py @@ -15,6 +15,7 @@ from deepmd.utils.type_embed import embed_atom_type from deepmd.utils.sess import run_sess from deepmd.utils.graph import load_graph_def, get_tensor_by_name_from_graph, get_tensor_by_name +from deepmd.utils.graph import get_attention_layer_variables_from_graph_def from deepmd.utils.errors import GraphWithoutTensorError from .descriptor import Descriptor from .se_a import DescrptSeA @@ -117,6 +118,9 @@ def __init__(self, self.sel_all_r = [0] avg_zero = np.zeros([self.ntypes, self.ndescrpt]).astype(GLOBAL_NP_FLOAT_PRECISION) std_ones = np.ones([self.ntypes, self.ndescrpt]).astype(GLOBAL_NP_FLOAT_PRECISION) + self.beta = np.zeros([self.attn_layer, self.filter_neuron[-1]]).astype(GLOBAL_NP_FLOAT_PRECISION) + self.gamma = np.ones([self.attn_layer, self.filter_neuron[-1]]).astype(GLOBAL_NP_FLOAT_PRECISION) + self.attention_layer_variables = None sub_graph = tf.Graph() with sub_graph.as_default(): name_pfx = 'd_sea_' @@ -305,10 +309,6 @@ def build(self, self.attn_weight = [None for i in range(self.attn_layer)] self.angular_weight = [None for i in range(self.attn_layer)] self.attn_weight_final = [None for i in range(self.attn_layer)] - self.G = None - self.qs = [None for i in range(self.attn_layer)] - self.ks = [None for i in range(self.attn_layer)] - self.vs = [None for i in range(self.attn_layer)] self.descrpt, self.descrpt_deriv, self.rij, self.nlist, self.nei_type_vec, self.nmask \ = op_module.prod_env_mat_a_mix(coord, @@ -365,8 +365,8 @@ def _pass_filter(self, inputs_i = inputs inputs_i = tf.reshape(inputs_i, [-1, self.ndescrpt]) type_i = -1 - layer, qmat = self._filter(inputs_i, type_i, natoms, name='filter_type_all' + suffix, reuse=reuse, - trainable=trainable, activation_fn=self.filter_activation_fn, + layer, qmat = self._filter(inputs_i, type_i, natoms, name='filter_type_all' + suffix, suffix=suffix, + reuse=reuse, trainable=trainable, activation_fn=self.filter_activation_fn, type_embedding=type_embedding, atype=atype) layer = tf.reshape(layer, [tf.shape(inputs)[0], natoms[0], self.get_dim_out()]) qmat = tf.reshape(qmat, [tf.shape(inputs)[0], natoms[0], self.get_dim_rot_mat_1() * 3]) @@ -508,7 +508,8 @@ def _feedforward(self, input_xyz, d_in, d_mid): activation_fn=None, precision=self.filter_precision, trainable=True, - uniform_seed=self.uniform_seed)) + uniform_seed=self.uniform_seed, + initial_variables=self.attention_layer_variables)) input_xyz = one_layer( input_xyz, d_in, @@ -518,7 +519,8 @@ def _feedforward(self, input_xyz, d_in, d_mid): activation_fn=None, precision=self.filter_precision, trainable=True, - uniform_seed=self.uniform_seed) + uniform_seed=self.uniform_seed, + initial_variables=self.attention_layer_variables) input_xyz += residual input_xyz = tf.keras.layers.LayerNormalization()(input_xyz) return input_xyz @@ -553,75 +555,75 @@ def _attention_layers( input_r, dotr=False, do_mask=False, - trainable=True + trainable=True, + suffix='' ): sd_k = tf.sqrt(tf.cast(1., dtype=self.filter_precision)) - self.G = tf.reshape(input_xyz, (-1, shape_i[1] // 4, outputs_size[-1]))[0] for i in range(layer_num): - with tf.variable_scope('attention_layer{}_'.format(i), reuse=tf.AUTO_REUSE): + name = 'attention_layer_{}{}'.format(i, suffix) + with tf.variable_scope(name, reuse=tf.AUTO_REUSE): # input_xyz_in = tf.nn.l2_normalize(input_xyz, -1) Q_c = one_layer( input_xyz, self.att_n, name='c_query', + scope=name+'/', reuse=tf.AUTO_REUSE, seed=self.seed, activation_fn=None, precision=self.filter_precision, trainable=trainable, - uniform_seed=self.uniform_seed) + uniform_seed=self.uniform_seed, + initial_variables=self.attention_layer_variables) K_c = one_layer( input_xyz, self.att_n, name='c_key', + scope=name+'/', reuse=tf.AUTO_REUSE, seed=self.seed, activation_fn=None, precision=self.filter_precision, trainable=trainable, - uniform_seed=self.uniform_seed) + uniform_seed=self.uniform_seed, + initial_variables=self.attention_layer_variables) V_c = one_layer( input_xyz, self.att_n, name='c_value', + scope=name+'/', reuse=tf.AUTO_REUSE, seed=self.seed, activation_fn=None, precision=self.filter_precision, trainable=trainable, - uniform_seed=self.uniform_seed) + uniform_seed=self.uniform_seed, + initial_variables=self.attention_layer_variables) # # natom x nei_type_i x out_size # xyz_scatter = tf.reshape(xyz_scatter, (-1, shape_i[1] // 4, outputs_size[-1])) # natom x nei_type_i x att_n Q_c = tf.nn.l2_normalize(tf.reshape(Q_c, (-1, shape_i[1] // 4, self.att_n)), -1) K_c = tf.nn.l2_normalize(tf.reshape(K_c, (-1, shape_i[1] // 4, self.att_n)), -1) V_c = tf.nn.l2_normalize(tf.reshape(V_c, (-1, shape_i[1] // 4, self.att_n)), -1) - # Q_c = tf.reshape(Q_c, (-1, shape_i[1] // 4, self.att_n)) - # K_c = tf.reshape(K_c, (-1, shape_i[1] // 4, self.att_n)) - # V_c = tf.reshape(V_c, (-1, shape_i[1] // 4, self.att_n)) - self.qs[i] = Q_c[0] - self.ks[i] = K_c[0] - self.vs[i] = V_c[0] input_att = self._scaled_dot_attn(Q_c, K_c, V_c, sd_k, input_r, dotr=dotr, do_mask=do_mask, layer=i) input_att = tf.reshape(input_att, (-1, self.att_n)) - # A_c = tf.nn.softmax(tf.matmul(Q_c, K_c, transpose_b=True)/sd_k) - # # (natom x nei_type_i) x att_n - # input_att = tf.reshape(tf.matmul(A_c, V_c), (-1, self.att_n)) - # (natom x nei_type_i) x out_size input_xyz += one_layer( input_att, outputs_size[-1], name='c_out', + scope=name+'/', reuse=tf.AUTO_REUSE, seed=self.seed, activation_fn=None, precision=self.filter_precision, trainable=trainable, - uniform_seed=self.uniform_seed) - input_xyz = tf.keras.layers.LayerNormalization()(input_xyz) + uniform_seed=self.uniform_seed, + initial_variables=self.attention_layer_variables) + input_xyz = tf.keras.layers.LayerNormalization(beta_initializer=tf.constant_initializer(self.beta[i]), + gamma_initializer=tf.constant_initializer(self.gamma[i]))(input_xyz) # input_xyz = self._feedforward(input_xyz, outputs_size[-1], self.att_n) return input_xyz @@ -688,7 +690,7 @@ def _filter_lower( # natom x nei_type_i x out_size xyz_scatter_att = tf.reshape( self._attention_layers(xyz_scatter, self.attn_layer, shape_i, outputs_size, input_r, - dotr=self.attn_dotr, do_mask=self.attn_mask, trainable=trainable), + dotr=self.attn_dotr, do_mask=self.attn_mask, trainable=trainable, suffix=suffix), (-1, shape_i[1] // 4, outputs_size[-1])) # xyz_scatter = tf.reshape(xyz_scatter, (-1, shape_i[1] // 4, outputs_size[-1])) else: @@ -712,6 +714,7 @@ def _filter( activation_fn=tf.nn.tanh, stddev=1.0, bavg=0.0, + suffix='', name='linear', reuse=None, trainable=True): @@ -745,6 +748,7 @@ def _filter( stddev=stddev, bavg=bavg, trainable=trainable, + suffix=suffix, name=name, reuse=reuse, atype=atype) @@ -775,3 +779,31 @@ def _filter( result = tf.reshape(result, [-1, outputs_size_2 * outputs_size[-1]]) return result, qmat + + def init_variables(self, + graph: tf.Graph, + graph_def: tf.GraphDef, + suffix: str = "", + ) -> None: + """ + Init the embedding net variables with the given dict + + Parameters + ---------- + graph : tf.Graph + The input frozen model graph + graph_def : tf.GraphDef + The input frozen model graph_def + suffix : str, optional + The suffix of the scope + """ + super().init_variables(graph=graph, graph_def=graph_def, suffix=suffix) + self.attention_layer_variables = get_attention_layer_variables_from_graph_def(graph_def, suffix=suffix) + if self.attn_layer > 0: + self.beta[0] = self.attention_layer_variables['attention_layer_0{}/layer_normalization/beta'.format(suffix)] + self.gamma[0] = self.attention_layer_variables['attention_layer_0{}/layer_normalization/gamma'.format(suffix)] + for i in range(1, self.attn_layer): + self.beta[i] = self.attention_layer_variables[ + 'attention_layer_{}{}/layer_normalization_{}/beta'.format(i, suffix, i)] + self.gamma[i] = self.attention_layer_variables[ + 'attention_layer_{}{}/layer_normalization_{}/gamma'.format(i, suffix, i)] diff --git a/deepmd/env.py b/deepmd/env.py index f0d6e1da4e..36c4b4b7c2 100644 --- a/deepmd/env.py +++ b/deepmd/env.py @@ -44,6 +44,8 @@ "TRANSFER_PATTERN", "FITTING_NET_PATTERN", "EMBEDDING_NET_PATTERN", + "TYPE_EMBEDDING_PATTERN", + "ATTENTION_LAYER_PATTERN", "TF_VERSION" ] @@ -59,18 +61,26 @@ r"filter_type_\d+/matrix_\d+_\d+|" r"filter_type_\d+/bias_\d+_\d+|" r"filter_type_\d+/idt_\d+_\d+|" + r"filter_type_all/matrix_\d+|" r"filter_type_all/matrix_\d+_\d+|" r"filter_type_all/matrix_\d+_\d+_\d+|" + r"filter_type_all/bias_\d+|" r"filter_type_all/bias_\d+_\d+|" r"filter_type_all/bias_\d+_\d+_\d+|" + r"filter_type_all/idt_\d+|" r"filter_type_all/idt_\d+_\d+|" ) FITTING_NET_PATTERN = str( + r"layer_\d+/matrix|" r"layer_\d+_type_\d+/matrix|" + r"layer_\d+/bias|" r"layer_\d+_type_\d+/bias|" + r"layer_\d+/idt|" r"layer_\d+_type_\d+/idt|" + r"final_layer/matrix|" r"final_layer_type_\d+/matrix|" + r"final_layer/bias|" r"final_layer_type_\d+/bias|" ) @@ -80,6 +90,21 @@ r"type_embed_net+/idt_\d+|" ) +ATTENTION_LAYER_PATTERN = str( + r"attention_layer_\d+/c_query/matrix|" + r"attention_layer_\d+/c_query/bias|" + r"attention_layer_\d+/c_key/matrix|" + r"attention_layer_\d+/c_key/bias|" + r"attention_layer_\d+/c_value/matrix|" + r"attention_layer_\d+/c_value/bias|" + r"attention_layer_\d+/c_out/matrix|" + r"attention_layer_\d+/c_out/bias|" + r"attention_layer_\d+/layer_normalization/beta|" + r"attention_layer_\d+/layer_normalization/gamma|" + r"attention_layer_\d+/layer_normalization_\d+/beta|" + r"attention_layer_\d+/layer_normalization_\d+/gamma|" +) + TRANSFER_PATTERN = \ EMBEDDING_NET_PATTERN + \ FITTING_NET_PATTERN + \ diff --git a/deepmd/fit/ener.py b/deepmd/fit/ener.py index 64258d88ec..47e9589cf3 100644 --- a/deepmd/fit/ener.py +++ b/deepmd/fit/ener.py @@ -9,6 +9,7 @@ from deepmd.utils.network import one_layer as one_layer_deepmd from deepmd.utils.type_embed import embed_atom_type from deepmd.utils.graph import get_fitting_net_variables_from_graph_def, load_graph_def, get_tensor_by_name_from_graph +from deepmd.utils.errors import GraphWithoutTensorError from deepmd.fit.fitting import Fitting from deepmd.env import global_cvt_2_tf_float @@ -400,6 +401,8 @@ def build (self, if input_dict is None: input_dict = {} bias_atom_e = self.bias_atom_e + type_embedding = input_dict.get('type_embedding', None) + atype = input_dict.get('atype', None) if self.numb_fparam > 0: if self.fparam_avg is None: self.fparam_avg = 0. @@ -418,9 +421,10 @@ def build (self, t_daparam = tf.constant(self.numb_aparam, name = 'daparam', dtype = tf.int32) - self.t_bias_atom_e = tf.get_variable('t_bias_atom_e', + if type_embedding is not None: + self.t_bias_atom_e = tf.get_variable('t_bias_atom_e', self.bias_atom_e.shape, - dtype=GLOBAL_TF_FLOAT_PRECISION, + dtype=self.fitting_precision, trainable=False, initializer=tf.constant_initializer(self.bias_atom_e)) if self.numb_fparam > 0: @@ -471,9 +475,7 @@ def build (self, aparam = tf.reshape(aparam, [-1, self.numb_aparam]) aparam = (aparam - t_aparam_avg) * t_aparam_istd aparam = tf.reshape(aparam, [-1, self.numb_aparam * natoms[0]]) - - type_embedding = input_dict.get('type_embedding', None) - atype = input_dict.get('atype', None) + if type_embedding is not None: atype_nall = tf.reshape(atype, [-1, natoms[1]]) self.atype_nloc = tf.reshape(tf.slice(atype_nall, [0, 0], [-1, natoms[0]]), [-1]) ## lammps will make error @@ -570,6 +572,11 @@ def init_variables(self, if self.numb_aparam > 0: self.aparam_avg = get_tensor_by_name_from_graph(graph, 'fitting_attr%s/t_aparam_avg' % suffix) self.aparam_inv_std = get_tensor_by_name_from_graph(graph, 'fitting_attr%s/t_aparam_istd' % suffix) + try: + self.bias_atom_e = get_tensor_by_name_from_graph(graph, 'fitting_attr%s/t_bias_atom_e' % suffix) + except GraphWithoutTensorError: + # model without type_embedding has no t_bias_atom_e + pass def enable_compression(self, model_file: str, diff --git a/deepmd/train/trainer.py b/deepmd/train/trainer.py index 52cee2427b..79038709c1 100644 --- a/deepmd/train/trainer.py +++ b/deepmd/train/trainer.py @@ -288,7 +288,8 @@ def _init_param(self, jdata): def build (self, data = None, - stop_batch = 0) : + stop_batch = 0, + suffix = "") : self.ntypes = self.model.get_ntypes() self.stop_batch = stop_batch @@ -348,7 +349,7 @@ def build (self, self.fitting.enable_mixed_precision(self.mixed_prec) self._build_lr() - self._build_network(data) + self._build_network(data, suffix) self._build_training() @@ -358,7 +359,7 @@ def _build_lr(self): self.learning_rate = self.lr.build(self.global_step, self.stop_batch) log.info("built lr") - def _build_network(self, data): + def _build_network(self, data, suffix=""): self.place_holders = {} if self.is_compress : for kk in ['coord', 'box']: @@ -379,7 +380,7 @@ def _build_network(self, data): self.place_holders['default_mesh'], self.place_holders, self.frz_model, - suffix = "", + suffix = suffix, reuse = False) self.l2_l, self.l2_more\ diff --git a/deepmd/utils/argcheck.py b/deepmd/utils/argcheck.py index 1f84b7fe60..ce7710d116 100644 --- a/deepmd/utils/argcheck.py +++ b/deepmd/utils/argcheck.py @@ -276,9 +276,9 @@ def descrpt_se_atten_args(): Argument("seed", [int, None], optional=True, doc=doc_seed), Argument("exclude_types", list, optional=True, default=[], doc=doc_exclude_types), Argument("set_davg_zero", bool, optional=True, default=False, doc=doc_set_davg_zero), - Argument("attn", int, optional=True, default=100, doc=doc_attn), - Argument("attn_layer", int, optional=True, default=4, doc=doc_attn_layer), - Argument("attn_dotr", bool, optional=True, default=False, doc=doc_attn_dotr), + Argument("attn", int, optional=True, default=128, doc=doc_attn), + Argument("attn_layer", int, optional=True, default=2, doc=doc_attn_layer), + Argument("attn_dotr", bool, optional=True, default=True, doc=doc_attn_dotr), Argument("attn_mask", bool, optional=True, default=False, doc=doc_attn_mask) ] diff --git a/deepmd/utils/graph.py b/deepmd/utils/graph.py index 10e13730c5..a8d95ebb25 100644 --- a/deepmd/utils/graph.py +++ b/deepmd/utils/graph.py @@ -1,7 +1,7 @@ import re import numpy as np from typing import Tuple, Dict -from deepmd.env import tf, EMBEDDING_NET_PATTERN, FITTING_NET_PATTERN, TYPE_EMBEDDING_PATTERN +from deepmd.env import tf, EMBEDDING_NET_PATTERN, FITTING_NET_PATTERN, TYPE_EMBEDDING_PATTERN, ATTENTION_LAYER_PATTERN from deepmd.utils.sess import run_sess from deepmd.utils.errors import GraphWithoutTensorError @@ -391,3 +391,63 @@ def get_type_embedding_net_variables_from_graph_def(graph_def: tf.GraphDef, suff tensor_value = get_tensor_by_type(node, dtype) type_embedding_net_variables[item] = np.reshape(tensor_value, tensor_shape) return type_embedding_net_variables + + +def get_attention_layer_nodes_from_graph_def(graph_def: tf.GraphDef, suffix: str = "") -> Dict: + """ + Get the attention layer nodes with the given tf.GraphDef object + + Parameters + ---------- + graph_def + The input tf.GraphDef object + suffix : str, optional + The scope suffix + + Returns + ---------- + Dict + The attention layer nodes within the given tf.GraphDef object + """ + if suffix != "": + attention_layer_pattern = ATTENTION_LAYER_PATTERN \ + .replace('/c_query', suffix + '/c_query') \ + .replace('/c_key', suffix + '/c_key') \ + .replace('/c_value', suffix + '/c_value') \ + .replace('/c_out', suffix + '/c_out') \ + .replace('/layer_normalization', suffix + '/layer_normalization') + else: + attention_layer_pattern = ATTENTION_LAYER_PATTERN + + attention_layer_nodes = get_pattern_nodes_from_graph_def(graph_def, attention_layer_pattern) + return attention_layer_nodes + + +def get_attention_layer_variables_from_graph_def(graph_def: tf.GraphDef, suffix: str = "") -> Dict: + """ + Get the attention layer variables with the given tf.GraphDef object + + Parameters + ---------- + graph_def : tf.GraphDef + The input tf.GraphDef object + suffix : str, optional + The suffix of the scope + + Returns + ---------- + Dict + The attention layer variables within the given tf.GraphDef object + """ + attention_layer_variables = {} + attention_layer_net_nodes = get_attention_layer_nodes_from_graph_def(graph_def, suffix=suffix) + for item in attention_layer_net_nodes: + node = attention_layer_net_nodes[item] + dtype = tf.as_dtype(node.dtype).as_numpy_dtype + tensor_shape = tf.TensorShape(node.tensor_shape).as_list() + if (len(tensor_shape) != 1) or (tensor_shape[0] != 1): + tensor_value = np.frombuffer(node.tensor_content, dtype=tf.as_dtype(node.dtype).as_numpy_dtype) + else: + tensor_value = get_tensor_by_type(node, dtype) + attention_layer_variables[item] = np.reshape(tensor_value, tensor_shape) + return attention_layer_variables diff --git a/deepmd/utils/network.py b/deepmd/utils/network.py index 22542adfba..47d9d83351 100644 --- a/deepmd/utils/network.py +++ b/deepmd/utils/network.py @@ -13,7 +13,8 @@ def one_layer(inputs, precision = GLOBAL_TF_FLOAT_PRECISION, stddev=1.0, bavg=0.0, - name='linear', + name='linear', + scope='', reuse=None, seed=None, use_timestep = False, @@ -36,8 +37,8 @@ def one_layer(inputs, mean=bavg, seed=seed if (seed is None or uniform_seed) else seed + 1) if initial_variables is not None: - w_initializer = tf.constant_initializer(initial_variables[name + '/matrix']) - b_initializer = tf.constant_initializer(initial_variables[name + '/bias']) + w_initializer = tf.constant_initializer(initial_variables[scope + name + '/matrix']) + b_initializer = tf.constant_initializer(initial_variables[scope + name + '/bias']) w = tf.get_variable('matrix', [shape[1], outputs_size], precision, @@ -63,7 +64,7 @@ def one_layer(inputs, mean=0.1, seed=seed if (seed is None or uniform_seed) else seed + 2) if initial_variables is not None: - idt_initializer = tf.constant_initializer(initial_variables[name + '/idt']) + idt_initializer = tf.constant_initializer(initial_variables[scope + name + '/idt']) idt = tf.get_variable('idt', [outputs_size], precision, diff --git a/source/tests/init_frz_model/data/set.000/box.npy b/source/tests/init_frz_model/data/set.000/box.npy new file mode 100644 index 0000000000000000000000000000000000000000..aa092a9aad05a63db85de60fba66177184c25934 GIT binary patch literal 200 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1ZlV+i=qoAIaUsO_*m=~X4l#&V(cT3DEP6dh= fXCxM+0{I%2I+{8PwF(pfE(S2ra)8jdNW$^}G$I_L literal 0 HcmV?d00001 diff --git a/source/tests/init_frz_model/data/set.000/coord.npy b/source/tests/init_frz_model/data/set.000/coord.npy new file mode 100644 index 0000000000000000000000000000000000000000..2205bfd45207e3c240c2b27877ddce45ece2e2d9 GIT binary patch literal 272 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1ZlV+i=qoAIaUsO_*m=~X4l#&V(cT3DEP6dh= zXCxM+0{I$-7CM?b3bhL411`;&{zG3^XgXAjtaE=@#o|zC%XNKm0iy$mC!(j}@J$1x zfX(4e;10Rg*Q^c-NQVD_pam(0gLqGvQGC{!JgFA9CNYm%Qh(XrIjE{axHUjQcbm?>}(T T`=s8Allygo4frR0I&TjEeotrA literal 0 HcmV?d00001 diff --git a/source/tests/init_frz_model/data/type.raw b/source/tests/init_frz_model/data/type.raw new file mode 100644 index 0000000000..e329bb5191 --- /dev/null +++ b/source/tests/init_frz_model/data/type.raw @@ -0,0 +1,6 @@ +0 +1 +1 +0 +1 +1 \ No newline at end of file diff --git a/source/tests/init_frz_model/data/type_map.raw b/source/tests/init_frz_model/data/type_map.raw new file mode 100644 index 0000000000..e900768b1d --- /dev/null +++ b/source/tests/init_frz_model/data/type_map.raw @@ -0,0 +1,2 @@ +O +H diff --git a/source/tests/init_frz_model/input.json b/source/tests/init_frz_model/input.json new file mode 100644 index 0000000000..4c4e93fb05 --- /dev/null +++ b/source/tests/init_frz_model/input.json @@ -0,0 +1,66 @@ +{ + "_comment": " model parameters", + "model": { +"type_map": ["O", "H"], +"descriptor" :{ + "type": "se_e2_a", + "sel": [46, 92], + "rcut_smth": 0.50, + "rcut": 6.00, + "neuron": [4, 8, 16], + "resnet_dt": false, + "axis_neuron": 16, + "seed": 1, + "_comment": " that's all" +}, +"fitting_net" : { + "neuron": [20, 20, 20], + "resnet_dt": true, + "seed": 1, + "_comment": " that's all" +}, +"_comment": " that's all" + }, + + "learning_rate" :{ +"type": "exp", +"decay_steps": 5000, +"start_lr": 0.001, +"stop_lr": 3.51e-8, +"_comment": "that's all" + }, + + "loss" :{ +"type": "ener", +"start_pref_e": 0.02, +"limit_pref_e": 1, +"start_pref_f": 1000, +"limit_pref_f": 1, +"start_pref_v": 1, +"limit_pref_v": 1, +"_comment": " that's all" + }, + + "training" : { +"training_data": { + "systems": ["init_frz_model/data"], + "batch_size": "auto", + "_comment": "that's all" +}, +"validation_data":{ + "systems": ["init_frz_model/data"], + "batch_size": 1, + "numb_btch": 3, + "_comment": "that's all" +}, +"numb_steps": 1, +"seed": 10, +"disp_file": "lcurve.out", +"disp_freq": 1, +"save_freq": 1, +"_comment": "that's all" + }, + + "_comment": "that's all" +} + diff --git a/source/tests/test_init_frz_model_se_a.py b/source/tests/test_init_frz_model_se_a.py new file mode 100644 index 0000000000..cad93dbeca --- /dev/null +++ b/source/tests/test_init_frz_model_se_a.py @@ -0,0 +1,161 @@ +import os, sys, platform, shutil, dpdata, json +import numpy as np +import unittest +import subprocess as sp +from common import j_loader, tests_path +from deepmd.train.trainer import DPTrainer +from deepmd.train.run_options import RunOptions +from deepmd.utils.argcheck import normalize +from deepmd.utils.compat import update_deepmd_input +from deepmd.utils.data_system import DeepmdDataSystem + +from deepmd.env import GLOBAL_NP_FLOAT_PRECISION, tf + +if GLOBAL_NP_FLOAT_PRECISION == np.float32: + default_places = 4 +else: + default_places = 10 + + +def _file_delete(file): + if os.path.isdir(file): + os.rmdir(file) + elif os.path.isfile(file): + os.remove(file) + + +def _subprocess_run(command): + popen = sp.Popen(command.split(), shell=False, stdout=sp.PIPE, stderr=sp.STDOUT) + for line in iter(popen.stdout.readline, b''): + if hasattr(line, 'decode'): + line = line.decode('utf-8') + line = line.rstrip() + print(line) + popen.wait() + return popen.returncode + + +def _init_models(): + data_file = str(tests_path / os.path.join("init_frz_model", "data")) + frozen_model = str(tests_path / "init_frz_se_a.pb") + ckpt = str(tests_path / "init_frz_se_a.ckpt") + run_opt_ckpt = RunOptions(init_model=ckpt, log_level=20) + run_opt_frz = RunOptions(init_frz_model=frozen_model, log_level=20) + INPUT = str(tests_path / "input.json") + jdata = j_loader(str(tests_path / os.path.join("init_frz_model", "input.json"))) + jdata["training"]["training_data"]["systems"] = data_file + jdata["training"]["validation_data"]["systems"] = data_file + jdata["training"]["save_ckpt"] = ckpt + with open(INPUT, "w") as fp: + json.dump(jdata, fp, indent=4) + ret = _subprocess_run("dp train " + INPUT) + np.testing.assert_equal(ret, 0, 'DP train failed!') + ret = _subprocess_run("dp freeze -c " + str(tests_path) + " -o " + frozen_model) + np.testing.assert_equal(ret, 0, 'DP freeze failed!') + + jdata = update_deepmd_input(jdata, warning=True, dump="input_v2_compat.json") + jdata = normalize(jdata) + model_ckpt = DPTrainer(jdata, run_opt=run_opt_ckpt) + model_frz = DPTrainer(jdata, run_opt=run_opt_frz) + rcut = model_ckpt.model.get_rcut() + type_map = model_ckpt.model.get_type_map() + data = DeepmdDataSystem( + systems=[data_file], + batch_size=1, + test_size=1, + rcut=rcut, + type_map=type_map, + trn_all_set=True + ) + data_requirement = {'energy': {'ndof': 1, + 'atomic': False, + 'must': False, + 'high_prec': True, + 'type_sel': None, + 'repeat': 1, + 'default': 0.0}, + 'force': {'ndof': 3, + 'atomic': True, + 'must': False, + 'high_prec': False, + 'type_sel': None, + 'repeat': 1, + 'default': 0.0}, + 'virial': {'ndof': 9, + 'atomic': False, + 'must': False, + 'high_prec': False, + 'type_sel': None, + 'repeat': 1, + 'default': 0.0}, + 'atom_ener': {'ndof': 1, + 'atomic': True, + 'must': False, + 'high_prec': False, + 'type_sel': None, + 'repeat': 1, + 'default': 0.0}, + 'atom_pref': {'ndof': 1, + 'atomic': True, + 'must': False, + 'high_prec': False, + 'type_sel': None, + 'repeat': 3, + 'default': 0.0}} + data.add_dict(data_requirement) + stop_batch = jdata["training"]["numb_steps"] + + return INPUT, ckpt, frozen_model, model_ckpt, model_frz, data, stop_batch + + +INPUT, CKPT, FROZEN_MODEL, CKPT_TRAINER, FRZ_TRAINER, VALID_DATA, STOP_BATCH = _init_models() + + +class TestInitFrzModelA(unittest.TestCase): + @classmethod + def setUpClass(self): + self.dp_ckpt = CKPT_TRAINER + self.dp_frz = FRZ_TRAINER + self.valid_data = VALID_DATA + self.stop_batch = STOP_BATCH + + @classmethod + def tearDownClass(self): + _file_delete(INPUT) + _file_delete(FROZEN_MODEL) + _file_delete("out.json") + _file_delete(str(tests_path / "checkpoint")) + _file_delete(CKPT+".meta") + _file_delete(CKPT+".index") + _file_delete(CKPT+".data-00000-of-00001") + _file_delete(CKPT+"-0.meta") + _file_delete(CKPT+"-0.index") + _file_delete(CKPT+"-0.data-00000-of-00001") + _file_delete(CKPT+"-1.meta") + _file_delete(CKPT+"-1.index") + _file_delete(CKPT+"-1.data-00000-of-00001") + _file_delete("input_v2_compat.json") + _file_delete("lcurve.out") + + def test_single_frame(self): + valid_batch = self.valid_data.get_batch() + natoms = valid_batch['natoms_vec'] + tf.reset_default_graph() + self.dp_ckpt.build(self.valid_data, self.stop_batch) + self.dp_ckpt._init_session() + feed_dict_ckpt = self.dp_ckpt.get_feed_dict(valid_batch, is_training=False) + ckpt_rmse_ckpt = self.dp_ckpt.loss.eval(self.dp_ckpt.sess, feed_dict_ckpt, natoms) + tf.reset_default_graph() + + self.dp_frz.build(self.valid_data, self.stop_batch) + self.dp_frz._init_session() + feed_dict_frz = self.dp_frz.get_feed_dict(valid_batch, is_training=False) + ckpt_rmse_frz = self.dp_frz.loss.eval(self.dp_frz.sess, feed_dict_frz, natoms) + tf.reset_default_graph() + + # check values + np.testing.assert_almost_equal(ckpt_rmse_ckpt['rmse_e'], ckpt_rmse_frz['rmse_e'], default_places) + np.testing.assert_almost_equal(ckpt_rmse_ckpt['rmse_f'], ckpt_rmse_frz['rmse_f'], default_places) + np.testing.assert_almost_equal(ckpt_rmse_ckpt['rmse_v'], ckpt_rmse_frz['rmse_v'], default_places) + + diff --git a/source/tests/test_init_frz_model_se_a_type.py b/source/tests/test_init_frz_model_se_a_type.py new file mode 100644 index 0000000000..2e7a693e3f --- /dev/null +++ b/source/tests/test_init_frz_model_se_a_type.py @@ -0,0 +1,165 @@ +import os, sys, platform, shutil, dpdata, json +import numpy as np +import unittest +import subprocess as sp +from common import j_loader, tests_path +from deepmd.train.trainer import DPTrainer +from deepmd.train.run_options import RunOptions +from deepmd.utils.argcheck import normalize +from deepmd.utils.compat import update_deepmd_input +from deepmd.utils.data_system import DeepmdDataSystem + +from deepmd.env import GLOBAL_NP_FLOAT_PRECISION, tf + +if GLOBAL_NP_FLOAT_PRECISION == np.float32: + default_places = 4 +else: + default_places = 10 + + +def _file_delete(file): + if os.path.isdir(file): + os.rmdir(file) + elif os.path.isfile(file): + os.remove(file) + + +def _subprocess_run(command): + popen = sp.Popen(command.split(), shell=False, stdout=sp.PIPE, stderr=sp.STDOUT) + for line in iter(popen.stdout.readline, b''): + if hasattr(line, 'decode'): + line = line.decode('utf-8') + line = line.rstrip() + print(line) + popen.wait() + return popen.returncode + + +def _init_models(): + data_file = str(tests_path / os.path.join("init_frz_model", "data")) + frozen_model = str(tests_path / "init_frz_se_a_type.pb") + ckpt = str(tests_path / "init_frz_se_a_type.ckpt") + run_opt_ckpt = RunOptions(init_model=ckpt, log_level=20) + run_opt_frz = RunOptions(init_frz_model=frozen_model, log_level=20) + INPUT = str(tests_path / "input.json") + jdata = j_loader(str(tests_path / os.path.join("init_frz_model", "input.json"))) + jdata["training"]["training_data"]["systems"] = data_file + jdata["training"]["validation_data"]["systems"] = data_file + jdata["training"]["save_ckpt"] = ckpt + type_embed = {} + type_embed['neuron'] = [2, 4, 8] + type_embed['resnet_dt'] = False + jdata['model']["type_embedding"] = type_embed + with open(INPUT, "w") as fp: + json.dump(jdata, fp, indent=4) + ret = _subprocess_run("dp train " + INPUT) + np.testing.assert_equal(ret, 0, 'DP train failed!') + ret = _subprocess_run("dp freeze -c " + str(tests_path) + " -o " + frozen_model) + np.testing.assert_equal(ret, 0, 'DP freeze failed!') + + jdata = update_deepmd_input(jdata, warning=True, dump="input_v2_compat.json") + jdata = normalize(jdata) + model_ckpt = DPTrainer(jdata, run_opt=run_opt_ckpt) + model_frz = DPTrainer(jdata, run_opt=run_opt_frz) + rcut = model_ckpt.model.get_rcut() + type_map = model_ckpt.model.get_type_map() + data = DeepmdDataSystem( + systems=[data_file], + batch_size=1, + test_size=1, + rcut=rcut, + type_map=type_map, + trn_all_set=True + ) + data_requirement = {'energy': {'ndof': 1, + 'atomic': False, + 'must': False, + 'high_prec': True, + 'type_sel': None, + 'repeat': 1, + 'default': 0.0}, + 'force': {'ndof': 3, + 'atomic': True, + 'must': False, + 'high_prec': False, + 'type_sel': None, + 'repeat': 1, + 'default': 0.0}, + 'virial': {'ndof': 9, + 'atomic': False, + 'must': False, + 'high_prec': False, + 'type_sel': None, + 'repeat': 1, + 'default': 0.0}, + 'atom_ener': {'ndof': 1, + 'atomic': True, + 'must': False, + 'high_prec': False, + 'type_sel': None, + 'repeat': 1, + 'default': 0.0}, + 'atom_pref': {'ndof': 1, + 'atomic': True, + 'must': False, + 'high_prec': False, + 'type_sel': None, + 'repeat': 3, + 'default': 0.0}} + data.add_dict(data_requirement) + stop_batch = jdata["training"]["numb_steps"] + + return INPUT, ckpt, frozen_model, model_ckpt, model_frz, data, stop_batch + + +INPUT, CKPT, FROZEN_MODEL, CKPT_TRAINER, FRZ_TRAINER, VALID_DATA, STOP_BATCH = _init_models() + + +class TestInitFrzModelAType(unittest.TestCase): + @classmethod + def setUpClass(self): + self.dp_ckpt = CKPT_TRAINER + self.dp_frz = FRZ_TRAINER + self.valid_data = VALID_DATA + self.stop_batch = STOP_BATCH + + @classmethod + def tearDownClass(self): + _file_delete(INPUT) + _file_delete(FROZEN_MODEL) + _file_delete("out.json") + _file_delete(str(tests_path / "checkpoint")) + _file_delete(CKPT+".meta") + _file_delete(CKPT+".index") + _file_delete(CKPT+".data-00000-of-00001") + _file_delete(CKPT+"-0.meta") + _file_delete(CKPT+"-0.index") + _file_delete(CKPT+"-0.data-00000-of-00001") + _file_delete(CKPT+"-1.meta") + _file_delete(CKPT+"-1.index") + _file_delete(CKPT+"-1.data-00000-of-00001") + _file_delete("input_v2_compat.json") + _file_delete("lcurve.out") + + def test_single_frame(self): + valid_batch = self.valid_data.get_batch() + natoms = valid_batch['natoms_vec'] + tf.reset_default_graph() + self.dp_ckpt.build(self.valid_data, self.stop_batch) + self.dp_ckpt._init_session() + feed_dict_ckpt = self.dp_ckpt.get_feed_dict(valid_batch, is_training=False) + ckpt_rmse_ckpt = self.dp_ckpt.loss.eval(self.dp_ckpt.sess, feed_dict_ckpt, natoms) + tf.reset_default_graph() + + self.dp_frz.build(self.valid_data, self.stop_batch) + self.dp_frz._init_session() + feed_dict_frz = self.dp_frz.get_feed_dict(valid_batch, is_training=False) + ckpt_rmse_frz = self.dp_frz.loss.eval(self.dp_frz.sess, feed_dict_frz, natoms) + tf.reset_default_graph() + + # check values + np.testing.assert_almost_equal(ckpt_rmse_ckpt['rmse_e'], ckpt_rmse_frz['rmse_e'], default_places) + np.testing.assert_almost_equal(ckpt_rmse_ckpt['rmse_f'], ckpt_rmse_frz['rmse_f'], default_places) + np.testing.assert_almost_equal(ckpt_rmse_ckpt['rmse_v'], ckpt_rmse_frz['rmse_v'], default_places) + + diff --git a/source/tests/test_init_frz_model_se_atten.py b/source/tests/test_init_frz_model_se_atten.py new file mode 100644 index 0000000000..0decb022e9 --- /dev/null +++ b/source/tests/test_init_frz_model_se_atten.py @@ -0,0 +1,167 @@ +import os, sys, platform, shutil, dpdata, json +import numpy as np +import unittest +import subprocess as sp +from common import j_loader, tests_path +from deepmd.train.trainer import DPTrainer +from deepmd.train.run_options import RunOptions +from deepmd.utils.argcheck import normalize +from deepmd.utils.compat import update_deepmd_input +from deepmd.utils.data_system import DeepmdDataSystem + +from deepmd.env import GLOBAL_NP_FLOAT_PRECISION, tf +from packaging.version import parse as parse_version + +if GLOBAL_NP_FLOAT_PRECISION == np.float32: + default_places = 4 +else: + default_places = 10 + + +def _file_delete(file): + if os.path.isdir(file): + os.rmdir(file) + elif os.path.isfile(file): + os.remove(file) + + +def _subprocess_run(command): + popen = sp.Popen(command.split(), shell=False, stdout=sp.PIPE, stderr=sp.STDOUT) + for line in iter(popen.stdout.readline, b''): + if hasattr(line, 'decode'): + line = line.decode('utf-8') + line = line.rstrip() + print(line) + popen.wait() + return popen.returncode + + +def _init_models(): + data_file = str(tests_path / os.path.join("init_frz_model", "data")) + frozen_model = str(tests_path / "init_frz_se_atten.pb") + ckpt = str(tests_path / "init_frz_se_atten.ckpt") + run_opt_ckpt = RunOptions(init_model=ckpt, log_level=20) + run_opt_frz = RunOptions(init_frz_model=frozen_model, log_level=20) + INPUT = str(tests_path / "input.json") + jdata = j_loader(str(tests_path / os.path.join("init_frz_model", "input.json"))) + jdata["training"]["training_data"]["systems"] = data_file + jdata["training"]["validation_data"]["systems"] = data_file + jdata["training"]["save_ckpt"] = ckpt + jdata['model']["descriptor"]['type'] = 'se_atten' + jdata['model']["descriptor"]['sel'] = 120 + with open(INPUT, "w") as fp: + json.dump(jdata, fp, indent=4) + ret = _subprocess_run("dp train " + INPUT) + np.testing.assert_equal(ret, 0, 'DP train failed!') + ret = _subprocess_run("dp freeze -c " + str(tests_path) + " -o " + frozen_model) + np.testing.assert_equal(ret, 0, 'DP freeze failed!') + + jdata = update_deepmd_input(jdata, warning=True, dump="input_v2_compat.json") + jdata = normalize(jdata) + model_ckpt = DPTrainer(jdata, run_opt=run_opt_ckpt) + model_frz = DPTrainer(jdata, run_opt=run_opt_frz) + rcut = model_ckpt.model.get_rcut() + type_map = model_ckpt.model.get_type_map() + data = DeepmdDataSystem( + systems=[data_file], + batch_size=1, + test_size=1, + rcut=rcut, + type_map=type_map, + trn_all_set=True + ) + data_requirement = {'energy': {'ndof': 1, + 'atomic': False, + 'must': False, + 'high_prec': True, + 'type_sel': None, + 'repeat': 1, + 'default': 0.0}, + 'force': {'ndof': 3, + 'atomic': True, + 'must': False, + 'high_prec': False, + 'type_sel': None, + 'repeat': 1, + 'default': 0.0}, + 'virial': {'ndof': 9, + 'atomic': False, + 'must': False, + 'high_prec': False, + 'type_sel': None, + 'repeat': 1, + 'default': 0.0}, + 'atom_ener': {'ndof': 1, + 'atomic': True, + 'must': False, + 'high_prec': False, + 'type_sel': None, + 'repeat': 1, + 'default': 0.0}, + 'atom_pref': {'ndof': 1, + 'atomic': True, + 'must': False, + 'high_prec': False, + 'type_sel': None, + 'repeat': 3, + 'default': 0.0}} + data.add_dict(data_requirement) + stop_batch = jdata["training"]["numb_steps"] + + return INPUT, ckpt, frozen_model, model_ckpt, model_frz, data, stop_batch + + +if not parse_version(tf.__version__) < parse_version("1.15"): + INPUT, CKPT, FROZEN_MODEL, CKPT_TRAINER, FRZ_TRAINER, VALID_DATA, STOP_BATCH = _init_models() + + +@unittest.skipIf(parse_version(tf.__version__) < parse_version("1.15"), + f"The current tf version {tf.__version__} is too low to run the new testing model.") +class TestInitFrzModelAtten(unittest.TestCase): + @classmethod + def setUpClass(self): + self.dp_ckpt = CKPT_TRAINER + self.dp_frz = FRZ_TRAINER + self.valid_data = VALID_DATA + self.stop_batch = STOP_BATCH + + @classmethod + def tearDownClass(self): + _file_delete(INPUT) + _file_delete(FROZEN_MODEL) + _file_delete("out.json") + _file_delete(str(tests_path / "checkpoint")) + _file_delete(CKPT+".meta") + _file_delete(CKPT+".index") + _file_delete(CKPT+".data-00000-of-00001") + _file_delete(CKPT+"-0.meta") + _file_delete(CKPT+"-0.index") + _file_delete(CKPT+"-0.data-00000-of-00001") + _file_delete(CKPT+"-1.meta") + _file_delete(CKPT+"-1.index") + _file_delete(CKPT+"-1.data-00000-of-00001") + _file_delete("input_v2_compat.json") + _file_delete("lcurve.out") + + def test_single_frame(self): + valid_batch = self.valid_data.get_batch() + natoms = valid_batch['natoms_vec'] + tf.reset_default_graph() + self.dp_ckpt.build(self.valid_data, self.stop_batch) + self.dp_ckpt._init_session() + feed_dict_ckpt = self.dp_ckpt.get_feed_dict(valid_batch, is_training=False) + ckpt_rmse_ckpt = self.dp_ckpt.loss.eval(self.dp_ckpt.sess, feed_dict_ckpt, natoms) + tf.reset_default_graph() + + self.dp_frz.build(self.valid_data, self.stop_batch) + self.dp_frz._init_session() + feed_dict_frz = self.dp_frz.get_feed_dict(valid_batch, is_training=False) + ckpt_rmse_frz = self.dp_frz.loss.eval(self.dp_frz.sess, feed_dict_frz, natoms) + tf.reset_default_graph() + + # check values + np.testing.assert_almost_equal(ckpt_rmse_ckpt['rmse_e'], ckpt_rmse_frz['rmse_e'], default_places) + np.testing.assert_almost_equal(ckpt_rmse_ckpt['rmse_f'], ckpt_rmse_frz['rmse_f'], default_places) + np.testing.assert_almost_equal(ckpt_rmse_ckpt['rmse_v'], ckpt_rmse_frz['rmse_v'], default_places) + + diff --git a/source/tests/test_init_frz_model_se_r.py b/source/tests/test_init_frz_model_se_r.py new file mode 100644 index 0000000000..f74a0ea222 --- /dev/null +++ b/source/tests/test_init_frz_model_se_r.py @@ -0,0 +1,167 @@ +import os, sys, platform, shutil, dpdata, json +import numpy as np +import unittest +import subprocess as sp +from common import j_loader, tests_path +from deepmd.train.trainer import DPTrainer +from deepmd.train.run_options import RunOptions +from deepmd.utils.argcheck import normalize +from deepmd.utils.compat import update_deepmd_input +from deepmd.utils.data_system import DeepmdDataSystem + +from deepmd.env import GLOBAL_NP_FLOAT_PRECISION, tf + +if GLOBAL_NP_FLOAT_PRECISION == np.float32: + default_places = 4 +else: + default_places = 10 + + +def _file_delete(file): + if os.path.isdir(file): + os.rmdir(file) + elif os.path.isfile(file): + os.remove(file) + + +def _subprocess_run(command): + popen = sp.Popen(command.split(), shell=False, stdout=sp.PIPE, stderr=sp.STDOUT) + for line in iter(popen.stdout.readline, b''): + if hasattr(line, 'decode'): + line = line.decode('utf-8') + line = line.rstrip() + print(line) + popen.wait() + return popen.returncode + + +def _init_models(): + data_file = str(tests_path / os.path.join("init_frz_model", "data")) + frozen_model = str(tests_path / "init_frz_se_r.pb") + ckpt = str(tests_path / "init_frz_se_r.ckpt") + run_opt_ckpt = RunOptions(init_model=ckpt, log_level=20) + run_opt_frz = RunOptions(init_frz_model=frozen_model, log_level=20) + INPUT = str(tests_path / "input.json") + jdata = j_loader(str(tests_path / os.path.join("init_frz_model", "input.json"))) + jdata["model"]["descriptor"] = {} + jdata["model"]["descriptor"]["type"] = "se_e2_r" + jdata["model"]["descriptor"]["sel"] = [46, 92] + jdata["model"]["descriptor"]["rcut_smth"] = 0.5 + jdata["model"]["descriptor"]["rcut"] = 6.0 + jdata["model"]["descriptor"]["neuron"] = [5, 10, 20] + jdata["model"]["descriptor"]["resnet_dt"] = False + jdata["model"]["descriptor"]["seed"] = 1 + jdata["training"]["training_data"]["systems"] = data_file + jdata["training"]["validation_data"]["systems"] = data_file + jdata["training"]["save_ckpt"] = ckpt + with open(INPUT, "w") as fp: + json.dump(jdata, fp, indent=4) + ret = _subprocess_run("dp train " + INPUT) + np.testing.assert_equal(ret, 0, 'DP train failed!') + ret = _subprocess_run("dp freeze -c " + str(tests_path) + " -o " + frozen_model) + np.testing.assert_equal(ret, 0, 'DP freeze failed!') + + jdata = update_deepmd_input(jdata, warning=True, dump="input_v2_compat.json") + jdata = normalize(jdata) + model_ckpt = DPTrainer(jdata, run_opt=run_opt_ckpt) + model_frz = DPTrainer(jdata, run_opt=run_opt_frz) + rcut = model_ckpt.model.get_rcut() + type_map = model_ckpt.model.get_type_map() + data = DeepmdDataSystem( + systems=[data_file], + batch_size=1, + test_size=1, + rcut=rcut, + type_map=type_map, + trn_all_set=True + ) + data_requirement = {'energy': {'ndof': 1, + 'atomic': False, + 'must': False, + 'high_prec': True, + 'type_sel': None, + 'repeat': 1, + 'default': 0.0}, + 'force': {'ndof': 3, + 'atomic': True, + 'must': False, + 'high_prec': False, + 'type_sel': None, + 'repeat': 1, + 'default': 0.0}, + 'virial': {'ndof': 9, + 'atomic': False, + 'must': False, + 'high_prec': False, + 'type_sel': None, + 'repeat': 1, + 'default': 0.0}, + 'atom_ener': {'ndof': 1, + 'atomic': True, + 'must': False, + 'high_prec': False, + 'type_sel': None, + 'repeat': 1, + 'default': 0.0}, + 'atom_pref': {'ndof': 1, + 'atomic': True, + 'must': False, + 'high_prec': False, + 'type_sel': None, + 'repeat': 3, + 'default': 0.0}} + data.add_dict(data_requirement) + stop_batch = jdata["training"]["numb_steps"] + + return INPUT, ckpt, frozen_model, model_ckpt, model_frz, data, stop_batch + + +INPUT, CKPT, FROZEN_MODEL, CKPT_TRAINER, FRZ_TRAINER, VALID_DATA, STOP_BATCH = _init_models() + + +class TestInitFrzModelR(unittest.TestCase): + @classmethod + def setUpClass(self): + self.dp_ckpt = CKPT_TRAINER + self.dp_frz = FRZ_TRAINER + self.valid_data = VALID_DATA + self.stop_batch = STOP_BATCH + + @classmethod + def tearDownClass(self): + _file_delete(INPUT) + _file_delete(FROZEN_MODEL) + _file_delete("out.json") + _file_delete(str(tests_path / "checkpoint")) + _file_delete(CKPT+".meta") + _file_delete(CKPT+".index") + _file_delete(CKPT+".data-00000-of-00001") + _file_delete(CKPT+"-0.meta") + _file_delete(CKPT+"-0.index") + _file_delete(CKPT+"-0.data-00000-of-00001") + _file_delete(CKPT+"-1.meta") + _file_delete(CKPT+"-1.index") + _file_delete(CKPT+"-1.data-00000-of-00001") + _file_delete("input_v2_compat.json") + _file_delete("lcurve.out") + + def test_single_frame(self): + valid_batch = self.valid_data.get_batch() + natoms = valid_batch['natoms_vec'] + tf.reset_default_graph() + self.dp_ckpt.build(self.valid_data, self.stop_batch) + self.dp_ckpt._init_session() + feed_dict_ckpt = self.dp_ckpt.get_feed_dict(valid_batch, is_training=False) + ckpt_rmse_ckpt = self.dp_ckpt.loss.eval(self.dp_ckpt.sess, feed_dict_ckpt, natoms) + tf.reset_default_graph() + + self.dp_frz.build(self.valid_data, self.stop_batch) + self.dp_frz._init_session() + feed_dict_frz = self.dp_frz.get_feed_dict(valid_batch, is_training=False) + ckpt_rmse_frz = self.dp_frz.loss.eval(self.dp_frz.sess, feed_dict_frz, natoms) + tf.reset_default_graph() + + # check values + np.testing.assert_almost_equal(ckpt_rmse_ckpt['rmse_e'], ckpt_rmse_frz['rmse_e'], default_places) + np.testing.assert_almost_equal(ckpt_rmse_ckpt['rmse_f'], ckpt_rmse_frz['rmse_f'], default_places) + np.testing.assert_almost_equal(ckpt_rmse_ckpt['rmse_v'], ckpt_rmse_frz['rmse_v'], default_places) From 7bcb3e2323021268929daf8fa848dbbd9e7ca718 Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Sun, 11 Sep 2022 22:45:40 -0400 Subject: [PATCH 13/21] add tutorial and publication links to docs (#1904) --- doc/index.rst | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/doc/index.rst b/doc/index.rst index 42fa0ec36b..b8b4d6862b 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -38,6 +38,17 @@ DeePMD-kit is a package written in Python/C++, designed to minimize the effort r nvnmd/index troubleshooting/index + +.. _tutorial: + +.. toctree:: + :maxdepth: 2 + :caption: Tutorial + :glob: + + Tutorials + Publications + .. _developer-guide: .. toctree:: From 716cb5d84a384189b56543c023bc3e5f9561b54f Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Mon, 12 Sep 2022 20:39:07 -0400 Subject: [PATCH 14/21] find protobuf headers from extra paths (#1910) fix #1905 --- source/cmake/Findtensorflow.cmake | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/source/cmake/Findtensorflow.cmake b/source/cmake/Findtensorflow.cmake index d6ef593968..97d21b8444 100644 --- a/source/cmake/Findtensorflow.cmake +++ b/source/cmake/Findtensorflow.cmake @@ -47,11 +47,17 @@ endif(DEFINED TENSORFLOW_ROOT) # define the search path list(APPEND TensorFlow_search_PATHS ${TENSORFLOW_ROOT}) -list(APPEND TensorFlow_search_PATHS "${TENSORFLOW_ROOT}/../tensorflow_core") +if(BUILD_CPP_IF) list(APPEND TensorFlow_search_PATHS ${TENSORFLOW_ROOT_NO64}) -list(APPEND TensorFlow_search_PATHS "${TENSORFLOW_ROOT_NO64}/../tensorflow_core") list(APPEND TensorFlow_search_PATHS "/usr/") list(APPEND TensorFlow_search_PATHS "/usr/local/") +endif() +if(BUILD_PY_IF) + # here TENSORFLOW_ROOT is path to site-packages/tensorflow + # for conda libraries, append extra paths + list(APPEND TensorFlow_search_PATHS "${TENSORFLOW_ROOT}/../tensorflow_core") + list(APPEND TensorFlow_search_PATHS "${TENSORFLOW_ROOT}/../../../..") +endif() # includes find_path(TensorFlow_INCLUDE_DIRS @@ -65,7 +71,6 @@ find_path(TensorFlow_INCLUDE_DIRS PATH_SUFFIXES "/include" NO_DEFAULT_PATH ) -if (BUILD_CPP_IF) find_path(TensorFlow_INCLUDE_DIRS_GOOGLE NAMES google/protobuf/type.pb.h @@ -74,11 +79,10 @@ find_path(TensorFlow_INCLUDE_DIRS_GOOGLE NO_DEFAULT_PATH ) list(APPEND TensorFlow_INCLUDE_DIRS ${TensorFlow_INCLUDE_DIRS_GOOGLE}) -endif () if (NOT TensorFlow_INCLUDE_DIRS AND tensorflow_FIND_REQUIRED) message(FATAL_ERROR - "Not found 'tensorflow/core/public/session.h' directory in path '${TensorFlow_search_PATHS}' " + "Not found 'include/tensorflow/core/public/session.h' directory or other header files in path '${TensorFlow_search_PATHS}' " "You can manually set the tensorflow install path by -DTENSORFLOW_ROOT ") endif () From 58bed2bb577985f888af0a4fbff1119203a346f1 Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Wed, 14 Sep 2022 03:51:41 -0400 Subject: [PATCH 15/21] =?UTF-8?q?LAMMPS:=20Use=20of=20=E2=80=9Coverride?= =?UTF-8?q?=E2=80=9D=20instead=20of=20=E2=80=9Cvirtual=E2=80=9D=20(#1915)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit fix #1855. --- source/lmp/compute_deeptensor_atom.h | 10 +++++----- source/lmp/fix_dplr.h | 22 +++++++++++----------- source/lmp/pair_deepmd.h.in | 22 +++++++++++----------- source/lmp/pppm_dplr.h | 10 +++++----- 4 files changed, 32 insertions(+), 32 deletions(-) diff --git a/source/lmp/compute_deeptensor_atom.h b/source/lmp/compute_deeptensor_atom.h index 60bafc5d7c..40d68f99f9 100644 --- a/source/lmp/compute_deeptensor_atom.h +++ b/source/lmp/compute_deeptensor_atom.h @@ -20,11 +20,11 @@ namespace LAMMPS_NS { class ComputeDeeptensorAtom : public Compute { public: ComputeDeeptensorAtom(class LAMMPS *, int, char **); - ~ComputeDeeptensorAtom(); - void init(); - void compute_peratom(); - double memory_usage(); - void init_list(int, class NeighList *); + ~ComputeDeeptensorAtom() override; + void init() override; + void compute_peratom() override; + double memory_usage() override; + void init_list(int, class NeighList *) override; private: int nmax; diff --git a/source/lmp/fix_dplr.h b/source/lmp/fix_dplr.h index c5cf5d22da..3367285ed3 100644 --- a/source/lmp/fix_dplr.h +++ b/source/lmp/fix_dplr.h @@ -29,17 +29,17 @@ namespace LAMMPS_NS { class FixDPLR : public Fix { public: FixDPLR(class LAMMPS *, int, char **); - virtual ~FixDPLR() {}; - int setmask(); - void init(); - void setup(int); - void post_integrate(); - void pre_force(int); - void post_force(int); - int pack_reverse_comm(int, int, double *); - void unpack_reverse_comm(int, int *, double *); - double compute_scalar(void); - double compute_vector(int); + ~FixDPLR() override {}; + int setmask() override; + void init() override; + void setup(int) override; + void post_integrate() override; + void pre_force(int) override; + void post_force(int) override; + int pack_reverse_comm(int, int, double *) override; + void unpack_reverse_comm(int, int *, double *) override; + double compute_scalar(void) override; + double compute_vector(int) override; private: PairDeepMD * pair_deepmd; deepmd::DeepTensor dpt; diff --git a/source/lmp/pair_deepmd.h.in b/source/lmp/pair_deepmd.h.in index 9b7480272b..45bcc03888 100644 --- a/source/lmp/pair_deepmd.h.in +++ b/source/lmp/pair_deepmd.h.in @@ -48,17 +48,17 @@ namespace LAMMPS_NS { class PairDeepMD : public Pair { public: PairDeepMD(class LAMMPS *); - virtual ~PairDeepMD(); - virtual void compute(int, int); - virtual void *extract(const char *, int &); - void settings(int, char **); - virtual void coeff(int, char **); - void init_style(); - virtual void write_restart(FILE *); - virtual void read_restart(FILE *); - double init_one(int i, int j); - int pack_reverse_comm(int, int, double *); - void unpack_reverse_comm(int, int *, double *); + ~PairDeepMD() override; + void compute(int, int) override; + void *extract(const char *, int &) override; + void settings(int, char **) override; + void coeff(int, char **) override; + void init_style() override; + void write_restart(FILE *) override; + void read_restart(FILE *) override; + double init_one(int i, int j) override; + int pack_reverse_comm(int, int, double *) override; + void unpack_reverse_comm(int, int *, double *) override; void print_summary(const std::string pre) const; int get_node_rank(); std::string get_file_content(const std::string & model); diff --git a/source/lmp/pppm_dplr.h b/source/lmp/pppm_dplr.h index 20e4107385..1a19aea258 100644 --- a/source/lmp/pppm_dplr.h +++ b/source/lmp/pppm_dplr.h @@ -27,13 +27,13 @@ namespace LAMMPS_NS { #else PPPMDPLR(class LAMMPS *); #endif - virtual ~PPPMDPLR () {}; - void init(); + ~PPPMDPLR () override {}; + void init() override; const std::vector & get_fele() const {return fele;}; protected: - virtual void compute(int, int); - virtual void fieldforce_ik(); - virtual void fieldforce_ad(); + void compute(int, int) override; + void fieldforce_ik() override; + void fieldforce_ad() override; private: std::vector fele; }; From 6944885937dba7c29868cdb03bfe3ccdac09fc5d Mon Sep 17 00:00:00 2001 From: Denghui Lu Date: Mon, 19 Sep 2022 11:44:28 +0800 Subject: [PATCH 16/21] Use GeluCustom as operator name (#1918) This should avoid the conflicts related to the custom `Gelu` and Tensorflow's `Gelu`. Also Fix #1913. --- deepmd/common.py | 4 +- source/op/_gelu.py | 13 ++++- source/op/gelu_multi_device.cc | 74 ++++++++++++++++++------- source/tests/test_activation_fn_gelu.py | 53 ++++++++++++++++++ 4 files changed, 121 insertions(+), 23 deletions(-) create mode 100644 source/tests/test_activation_fn_gelu.py diff --git a/deepmd/common.py b/deepmd/common.py index 6129011bbc..76c76bbf1e 100644 --- a/deepmd/common.py +++ b/deepmd/common.py @@ -66,7 +66,7 @@ def gelu(x: tf.Tensor) -> tf.Tensor: Original paper https://arxiv.org/abs/1606.08415 """ - return op_module.gelu(x) + return op_module.gelu_custom(x) def gelu_tf(x: tf.Tensor) -> tf.Tensor: @@ -94,7 +94,7 @@ def gelu_wrapper(x): return tensorflow.nn.gelu(x, approximate=True) except AttributeError: warnings.warn("TensorFlow does not provide an implementation of gelu, please upgrade your TensorFlow version. Fallback to the custom gelu operator.") - return op_module.gelu(x) + return op_module.gelu_custom(x) return (lambda x: gelu_wrapper(x))(x) # TODO this is not a good way to do things. This is some global variable to which diff --git a/source/op/_gelu.py b/source/op/_gelu.py index db45ef798e..dc818c1804 100644 --- a/source/op/_gelu.py +++ b/source/op/_gelu.py @@ -11,8 +11,17 @@ except AttributeError: @ops.RegisterGradient("Gelu") def _gelu_cc (op, dy) : - return op_module.gelu_grad(dy, op.inputs[0]) + return op_module.gelu_grad_custom(dy, op.inputs[0]) @ops.RegisterGradient("GeluGrad") def _gelu_grad_cc (op, dy) : - return [op_module.gelu_grad(dy, op.inputs[1]), op_module.gelu_grad_grad(dy, op.inputs[0], op.inputs[1])] + return [op_module.gelu_grad_custom(dy, op.inputs[1]), op_module.gelu_grad_grad_custom(dy, op.inputs[0], op.inputs[1])] + + +@ops.RegisterGradient("GeluCustom") +def _gelu_custom_cc (op, dy): + return op_module.gelu_grad_custom(dy, op.inputs[0]) + +@ops.RegisterGradient("GeluGradCustom") +def _gelu_grad_custom_cc (op, dy) : + return [op_module.gelu_grad_custom(dy, op.inputs[1]), op_module.gelu_grad_grad_custom(dy, op.inputs[0], op.inputs[1])] \ No newline at end of file diff --git a/source/op/gelu_multi_device.cc b/source/op/gelu_multi_device.cc index 9865599b4b..d1c9974b70 100644 --- a/source/op/gelu_multi_device.cc +++ b/source/op/gelu_multi_device.cc @@ -19,6 +19,24 @@ REGISTER_OP("GeluGradGrad") .Input("x: T") .Output("output: T"); +REGISTER_OP("GeluCustom") + .Attr("T: {float, double} = DT_DOUBLE") + .Input("x: T") + .Output("output: T"); + +REGISTER_OP("GeluGradCustom") + .Attr("T: {float, double} = DT_DOUBLE") + .Input("dy: T") + .Input("x: T") + .Output("output: T"); + +REGISTER_OP("GeluGradGradCustom") + .Attr("T: {float, double} = DT_DOUBLE") + .Input("dy: T") + .Input("dy_: T") + .Input("x: T") + .Output("output: T"); + // OpKernel definition. // template parameter is the datatype of the tensors. template @@ -178,29 +196,47 @@ class GeluGradGradOp : public OpKernel { std::string device; }; -#define REGISTER_CPU(T) \ -REGISTER_KERNEL_BUILDER( \ - Name("Gelu").Device(DEVICE_CPU).TypeConstraint("T"), \ - GeluOp); \ -REGISTER_KERNEL_BUILDER( \ - Name("GeluGrad").Device(DEVICE_CPU).TypeConstraint("T"), \ - GeluGradOp); \ -REGISTER_KERNEL_BUILDER( \ - Name("GeluGradGrad").Device(DEVICE_CPU).TypeConstraint("T"), \ - GeluGradGradOp); +#define REGISTER_CPU(T) \ +REGISTER_KERNEL_BUILDER( \ + Name("Gelu").Device(DEVICE_CPU).TypeConstraint("T"), \ + GeluOp); \ +REGISTER_KERNEL_BUILDER( \ + Name("GeluGrad").Device(DEVICE_CPU).TypeConstraint("T"), \ + GeluGradOp); \ +REGISTER_KERNEL_BUILDER( \ + Name("GeluGradGrad").Device(DEVICE_CPU).TypeConstraint("T"), \ + GeluGradGradOp); \ +REGISTER_KERNEL_BUILDER( \ + Name("GeluCustom").Device(DEVICE_CPU).TypeConstraint("T"), \ + GeluOp); \ +REGISTER_KERNEL_BUILDER( \ + Name("GeluGradCustom").Device(DEVICE_CPU).TypeConstraint("T"), \ + GeluGradOp); \ +REGISTER_KERNEL_BUILDER( \ + Name("GeluGradGradCustom").Device(DEVICE_CPU).TypeConstraint("T"), \ + GeluGradGradOp); REGISTER_CPU(float); REGISTER_CPU(double); #if GOOGLE_CUDA || TENSORFLOW_USE_ROCM -#define REGISTER_GPU(T) \ -REGISTER_KERNEL_BUILDER( \ - Name("Gelu").Device(DEVICE_GPU).TypeConstraint("T"), \ - GeluOp); \ -REGISTER_KERNEL_BUILDER( \ - Name("GeluGrad").Device(DEVICE_GPU).TypeConstraint("T"), \ - GeluGradOp); \ -REGISTER_KERNEL_BUILDER( \ - Name("GeluGradGrad").Device(DEVICE_GPU).TypeConstraint("T"), \ +#define REGISTER_GPU(T) \ +REGISTER_KERNEL_BUILDER( \ + Name("Gelu").Device(DEVICE_GPU).TypeConstraint("T"), \ + GeluOp); \ +REGISTER_KERNEL_BUILDER( \ + Name("GeluGrad").Device(DEVICE_GPU).TypeConstraint("T"), \ + GeluGradOp); \ +REGISTER_KERNEL_BUILDER( \ + Name("GeluGradGrad").Device(DEVICE_GPU).TypeConstraint("T"), \ + GeluGradGradOp); \ +REGISTER_KERNEL_BUILDER( \ + Name("GeluCustom").Device(DEVICE_GPU).TypeConstraint("T"), \ + GeluOp); \ +REGISTER_KERNEL_BUILDER( \ + Name("GeluGradCustom").Device(DEVICE_GPU).TypeConstraint("T"), \ + GeluGradOp); \ +REGISTER_KERNEL_BUILDER( \ + Name("GeluGradGradCustom").Device(DEVICE_GPU).TypeConstraint("T"), \ GeluGradGradOp); REGISTER_GPU(float); REGISTER_GPU(double); diff --git a/source/tests/test_activation_fn_gelu.py b/source/tests/test_activation_fn_gelu.py new file mode 100644 index 0000000000..cdf14072bc --- /dev/null +++ b/source/tests/test_activation_fn_gelu.py @@ -0,0 +1,53 @@ +import os,sys +import numpy as np +import unittest + +from deepmd.env import tf + +from deepmd.utils.network import embedding_net +from deepmd.common import get_activation_func + +class TestGelu(tf.test.TestCase): + def setUp (self) : + self.places = 6 + self.sess = self.test_session().__enter__() + self.inputs = tf.reshape(tf.constant([ 0., 1., 2., 3.], dtype = tf.float64), [-1, 1]) + self.refout = [[ 0.37703893, -0.38242253, -0.1862878, -0.23220415, 2.28706995, -0.40754364, + 0.22086098, -0.2690335 ], + [ 2.167494, 0.72560347, 0.99234317, 0.50832127, 5.20665818, 0.58361587, + 1.57217107, 0.67395218], + [ 4.19655852, 2.04779208, 2.20239826, 1.69247695, 8.38305924, 1.69006845, + 2.97176052, 1.76098426], + [ 6.21460216, 3.52613278, 3.39508271, 2.817003, 11.521799, 2.91028145, + 4.41870371, 2.82610791]] + + def test_activation_function_gelu_custom(self): + network_size = [2, 4, 8] + out = embedding_net(self.inputs, + network_size, + tf.float64, + activation_fn = get_activation_func('gelu'), + name_suffix = 'gelu_custom', + seed = 1, + uniform_seed = True) + self.sess.run(tf.global_variables_initializer()) + myout = self.sess.run(out) + np.testing.assert_almost_equal(self.refout, myout, self.places) + + + def test_activation_function_gelu_tensorflow(self): + network_size = [2, 4, 8] + out = embedding_net(self.inputs, + network_size, + tf.float64, + activation_fn = get_activation_func('gelu_tf'), + name_suffix = 'gelu_tensorflow', + seed = 1, + uniform_seed = True) + self.sess.run(tf.global_variables_initializer()) + myout = self.sess.run(out) + np.testing.assert_almost_equal(self.refout, myout, self.places) + + +if __name__ == '__main__': + unittest.main() From b396ca7dea25369ac5220578ad42f315eb81d278 Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Sun, 18 Sep 2022 23:46:21 -0400 Subject: [PATCH 17/21] handle float point error in sys_probs (#1919) Fix #1917. --- deepmd/utils/data_system.py | 9 ++++--- source/tests/test_deepmd_data_sys.py | 37 +++++++++++++++++++++++++++- 2 files changed, 41 insertions(+), 5 deletions(-) diff --git a/deepmd/utils/data_system.py b/deepmd/utils/data_system.py index a3782a906e..88578749f2 100644 --- a/deepmd/utils/data_system.py +++ b/deepmd/utils/data_system.py @@ -496,17 +496,18 @@ def _process_sys_probs(self, sys_probs) : sys_probs = np.array(sys_probs) type_filter = sys_probs >= 0 assigned_sum_prob = np.sum(type_filter * sys_probs) - assert assigned_sum_prob <= 1, "the sum of assigned probability should be less than 1" + # 1e-8 is to handle floating point error; See #1917 + assert assigned_sum_prob <= 1. + 1e-8, "the sum of assigned probability should be less than 1" rest_sum_prob = 1. - assigned_sum_prob - if rest_sum_prob != 0 : + if not np.isclose(rest_sum_prob, 0): rest_nbatch = (1 - type_filter) * self.nbatches rest_prob = rest_sum_prob * rest_nbatch / np.sum(rest_nbatch) ret_prob = rest_prob + type_filter * sys_probs else : ret_prob = sys_probs - assert np.sum(ret_prob) == 1, "sum of probs should be 1" + assert np.isclose(np.sum(ret_prob), 1), "sum of probs should be 1" return ret_prob - + def _prob_sys_size_ext(self, keywords): block_str = keywords.split(';')[1:] block_stt = [] diff --git a/source/tests/test_deepmd_data_sys.py b/source/tests/test_deepmd_data_sys.py index 5bab4d74e5..5582bdb4c2 100644 --- a/source/tests/test_deepmd_data_sys.py +++ b/source/tests/test_deepmd_data_sys.py @@ -289,4 +289,39 @@ def _in_array(self, target, idx_map, ndof, array): for idx,ii in enumerate(all_find) : self.assertTrue(ii, msg = 'does not find frame %d in array' % idx) - + def test_sys_prob_floating_point_error(self): + # test floating point error; See #1917 + sys_probs = [ + 0.010, + 0.010, + 0.010, + 0.010, + 0.010, + 0.010, + 0.010, + 0.010, + 0.010, + 0.150, + 0.100, + 0.100, + 0.050, + 0.050, + 0.020, + 0.015, + 0.015, + 0.050, + 0.020, + 0.015, + 0.040, + 0.055, + 0.025, + 0.025, + 0.015, + 0.025, + 0.055, + 0.040, + 0.040, + 0.005, + ] + ds = DeepmdDataSystem(self.sys_name, 3, 2, 2.0, sys_probs=sys_probs) + self.assertEqual(ds.sys_probs.size, len(sys_probs)) From c9b46e443688b1d6e722cb9a8b27b5d03c319f44 Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Mon, 19 Sep 2022 00:49:07 -0400 Subject: [PATCH 18/21] highlight LAMMPS codes in doc (#1921) See also https://github.com/lammps/pygments-lammps --- deepmd/nvnmd/entrypoints/wrap.py | 6 ++++-- doc/model/dplr.md | 14 +++++++------- doc/third-party/lammps-command.md | 22 +++++++++++----------- doc/third-party/lammps.md | 2 +- setup.py | 1 + 5 files changed, 24 insertions(+), 21 deletions(-) diff --git a/deepmd/nvnmd/entrypoints/wrap.py b/deepmd/nvnmd/entrypoints/wrap.py index 2e38c0d16d..d7a35b7a9f 100644 --- a/deepmd/nvnmd/entrypoints/wrap.py +++ b/deepmd/nvnmd/entrypoints/wrap.py @@ -19,8 +19,10 @@ class Wrap(): the model file can be use to run the NVNMD with lammps the pair style need set as: - | :code:`pair_style nvnmd model.pb` - | :code:`pair_coeff * *` + .. code-block:: lammps + + pair_style nvnmd model.pb + pair_coeff * * Parameters ---------- diff --git a/doc/model/dplr.md b/doc/model/dplr.md index 14db3facfc..4661cbcdf7 100644 --- a/doc/model/dplr.md +++ b/doc/model/dplr.md @@ -53,7 +53,7 @@ The training of the DPLR model is very similar to the standard short-range DP mo ``` The {ref}`model_name ` specifies which DW model is used to predict the position of WCs. {ref}`model_charge_map ` gives the amount of charge assigned to WCs. {ref}`sys_charge_map ` provides the nuclear charge of oxygen (type 0) and hydrogen (type 1) atoms. {ref}`ewald_beta ` (unit $\text{Å}^{-1}$) gives the spread parameter controls the spread of Gaussian charges, and {ref}`ewald_h ` (unit Å) assigns the grid size of Fourier transform. The DPLR model can be trained and frozen by (from the example directory) -``` +```bash dp train ener.json && dp freeze -o ener.pb ``` @@ -108,7 +108,7 @@ An example input script is provided in $deepmd_source_dir/examples/water/dplr/lmp/in.lammps ``` Here are some explanations -``` +```lammps # groups of real and virtual atoms group real_atom type 1 2 group virtual_atom type 3 @@ -124,7 +124,7 @@ special_bonds lj/coul 1 1 1 angle no ``` Type 1 and 2 (O and H) are `real_atom`s, while type 3 (WCs) are `virtual_atom`s. The model file `ener.pb` stores both the DW and DPLR models, so the position of WCs and the energy can be inferred from it. A virtual bond type is specified by `bond_style zero`. The `special_bonds` command switches off the exclusion of intramolecular interactions. -``` +```lammps # kspace_style "pppm/dplr" should be used. in addition the # gewald(1/distance) should be set the same as that used in # training. Currently only ik differentiation is supported. @@ -133,7 +133,7 @@ kspace_modify gewald ${BETA} diff ik mesh ${KMESH} ${KMESH} ${KMESH} ``` The long-range part is calculated by the `kspace` support of LAMMPS. The `kspace_style` `pppm/dplr` is required. The spread parameter set by variable `BETA` should be set the same as that used in training. The `KMESH` should be set dense enough so the long-range calculation is converged. -``` +```lammps # "fix dplr" set the position of the virtual atom, and spread the # electrostatic interaction asserting on the virtual atom to the real # atoms. "type_associate" associates the real atom type its @@ -144,7 +144,7 @@ fix_modify 0 virial yes ``` The fix command `dplr` calculates the position of WCs by the DW model and back-propagates the long-range interaction on virtual atoms to real toms. -``` +```lammps # compute the temperature of real atoms, excluding virtual atom contribution compute real_temp real_atom temp compute real_press all pressure real_temp @@ -152,12 +152,12 @@ fix 1 real_atom nvt temp ${TEMP} ${TEMP} ${TAU_T} fix_modify 1 temp real_temp ``` The temperature of the system should be computed from the real atoms. The kinetic contribution in the pressure tensor is also computed from the real atoms. The thermostat is applied to only real atoms. The computed temperature and pressure of real atoms can be accessed by, e.g. -``` +```lammps fix thermo_print all print ${THERMO_FREQ} "$(step) $(pe) $(ke) $(etotal) $(enthalpy) $(c_real_temp) $(c_real_press) $(vol) $(c_real_press[1]) $(c_real_press[2]) $(c_real_press[3])" append thermo.out screen no title "# step pe ke etotal enthalpy temp press vol pxx pyy pzz" ``` The LAMMPS simulation can be started from the example directory by -``` +```bash lmp -i in.lammps ``` If LAMMPS complains that no model file `ener.pb` exists, it can be copied from the training example directory. diff --git a/doc/third-party/lammps-command.md b/doc/third-party/lammps-command.md index c021837993..84eb5f5b99 100644 --- a/doc/third-party/lammps-command.md +++ b/doc/third-party/lammps-command.md @@ -4,7 +4,7 @@ If you are using the plugin mode, enable DeePMD-kit package in LAMMPS with `plugin` command: -``` +```lammps plugin load libdeepmd_lmp.so ``` @@ -22,7 +22,7 @@ The built-in mode doesn't need this step. The DeePMD-kit package provides the pair_style `deepmd` -``` +```lammps pair_style deepmd models ... keyword value ... ``` - deepmd = style of this pair_style @@ -44,7 +44,7 @@ and the model deviation will be computed among all models every `out_freq` times ### Examples -``` +```lammps pair_style deepmd graph.pb pair_style deepmd graph.pb fparam 1.2 pair_style deepmd graph_0.pb graph_1.pb graph_2.pb out_file md.out out_freq 10 atomic relative 1.0 @@ -71,7 +71,7 @@ where $D_{f_i}$ is the absolute model deviation of the force on atom $i$, $f_i$ The DeePMD-kit package provide the compute `deeptensor/atom` for computing atomic tensorial properties. -``` +```lammps compute ID group-ID deeptensor/atom model_file ``` - ID: user-assigned name of the computation @@ -80,11 +80,11 @@ compute ID group-ID deeptensor/atom model_file - model_file: the name of the binary model file. ### Examples -``` +```lammps compute dipole all deeptensor/atom dipole.pb ``` The result of the compute can be dump to trajctory file by -``` +```lammps dump 1 all custom 100 water.dump id type c_dipole[1] c_dipole[2] c_dipole[3] ``` @@ -94,7 +94,7 @@ dump 1 all custom 100 water.dump id type c_dipole[1] c_dipole[2] c_di ## Long-range interaction The reciprocal space part of the long-range interaction can be calculated by LAMMPS command `kspace_style`. To use it with DeePMD-kit, one writes -```bash +```lammps pair_style deepmd graph.pb pair_coeff * * kspace_style pppm 1.0e-5 @@ -111,13 +111,13 @@ $$dvatom=\sum_{m}( \mathbf{r}_n- \mathbf{r}_m) \frac{de_m}{d\mathbf{r}_n}$$ Where $\mathbf{r}_n$ is the atomic position of nth atom, $\mathbf{v}_n$ velocity of atom and $\frac{de_m}{d\mathbf{r}_n}$ the derivative of the atomic energy. In LAMMPS one can get the per-atom stress using the command `centroid/stress/atom`: -```bash +```lammps compute ID group-ID centroid/stress/atom NULL virial ``` see [LAMMPS doc page](https://docs.lammps.org/compute_stress_atom.html#thompson2) for more detailes on the meaning of the keywords. ### Examples In order of computing the 9-component per-atom stress -```bash +```lammps compute stress all centroid/stress/atom NULL virial ``` Thus `c_stress` is an array with 9 component in the order `xx,yy,zz,xy,xz,yz,yx,zx,zy`. @@ -130,7 +130,7 @@ Using per-atom stress tensor one can, for example, compute the heat flux defined $$\mathbf J = \sum_n e_n \mathbf v_n + \sum_{n,m} ( \mathbf r_m- \mathbf r_n) \frac{de_m}{d\mathbf r_n} \mathbf v_n$$ to compute the heat flux with LAMMPS: -```bash +```lammps compute ke_ID all ke/atom compute pe_ID all pe/atom compute stress_ID group-ID centroid/stress/atom NULL virial @@ -139,7 +139,7 @@ compute flux_ID all heat/flux ke_ID pe_ID stress_ID ### Examples -```bash +```lammps compute ke all ke/atom compute pe all pe/atom compute stress all centroid/stress/atom NULL virial diff --git a/doc/third-party/lammps.md b/doc/third-party/lammps.md index 78c6fc060e..0dc092c9e9 100644 --- a/doc/third-party/lammps.md +++ b/doc/third-party/lammps.md @@ -2,7 +2,7 @@ Running an MD simulation with LAMMPS is simpler. In the LAMMPS input file, one needs to specify the pair style as follows -``` +```lammps pair_style deepmd graph.pb pair_coeff * * ``` diff --git a/setup.py b/setup.py index edc7e0b97b..4803b043ee 100644 --- a/setup.py +++ b/setup.py @@ -153,6 +153,7 @@ "deepmodeling-sphinx>=0.1.0", "dargs>=0.3.1", "sphinx-argparse", + "pygments-lammps", ], **extras_require, }, From 5aab6d730a769e41d14ff2445b02dd992e4ebeee Mon Sep 17 00:00:00 2001 From: Duo <50307526+iProzd@users.noreply.github.com> Date: Thu, 22 Sep 2022 15:05:03 +0800 Subject: [PATCH 19/21] enforce type_map change when mixed_type (#1923) enforce type_map change when mixed_type. add out_of_bound check for 'real_atom_types.npy'. --- deepmd/utils/data.py | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/deepmd/utils/data.py b/deepmd/utils/data.py index b5faf15c34..4ce3d43d91 100644 --- a/deepmd/utils/data.py +++ b/deepmd/utils/data.py @@ -52,6 +52,9 @@ def __init__ (self, # load atom type self.atom_type = self._load_type(root) self.natoms = len(self.atom_type) + if self.mixed_type: + # nframes x natoms + self.atom_type_mix = self._load_type_mix(self.dirs[0]) # load atom type map self.type_map = self._load_type_map(root) if self.type_map is not None: @@ -63,6 +66,14 @@ def __init__ (self, if not self.mixed_type: atom_type_ = [type_map.index(self.type_map[ii]) for ii in self.atom_type] self.atom_type = np.array(atom_type_, dtype = np.int32) + else: + type_idx_map = np.searchsorted(type_map, self.type_map) + try: + atom_type_mix_ = np.array(type_idx_map)[self.atom_type_mix].astype(np.int32) + except RuntimeError as e: + raise RuntimeError("some types in 'real_atom_types.npy' of sys {} are not contained in {} types!" + .format(self.dirs[0], self.get_ntypes())) from e + self.atom_type_mix = atom_type_mix_ self.type_map = type_map if type_map is None and self.type_map is None and self.mixed_type: raise RuntimeError('mixed_type format must have type_map!') @@ -456,13 +467,17 @@ def _load_set(self, set_name: DPPath) : data[kk] = np.sum(np.reshape(tmp_in, [nframes, self.natoms, ndof]), axis = 1) if self.mixed_type: - type_path = set_name / "real_atom_types.npy" - real_type = type_path.load_numpy().astype(np.int32).reshape([nframes, -1]) + real_type = self.atom_type_mix.reshape([nframes, self.natoms]) data['type'] = real_type natoms = data['type'].shape[1] + # nframes x ntypes + atom_type_nums = np.array([(real_type == i).sum(axis=-1) for i in range(self.get_ntypes())], + dtype=np.int32).T + assert (atom_type_nums.sum(axis=-1) == natoms).all(), \ + "some types in 'real_atom_types.npy' of sys {} are not contained in {} types!" \ + .format(self.dirs[0], self.get_ntypes()) data['real_natoms_vec'] = np.concatenate((np.tile(np.array([natoms, natoms], dtype=np.int32), (nframes, 1)), - np.array([(real_type == i).sum(axis=-1) for i in range(self.get_ntypes())], - dtype=np.int32).T), axis=-1) + atom_type_nums), axis=-1) else: data['type'] = np.tile(self.atom_type[self.idx_map], (nframes, 1)) @@ -518,6 +533,11 @@ def _load_type (self, sys_path: DPPath) : atom_type = (sys_path / "type.raw").load_txt(dtype=np.int32, ndmin=1) return atom_type + def _load_type_mix(self, set_name: DPPath): + type_path = set_name / "real_atom_types.npy" + real_type = type_path.load_numpy().astype(np.int32).reshape([-1, self.natoms]) + return real_type + def _make_idx_map(self, atom_type): natoms = atom_type.shape[0] idx = np.arange (natoms) From f0973f08a7ed5edf33af7450fa80608d1c725c68 Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Thu, 22 Sep 2022 21:19:56 -0400 Subject: [PATCH 20/21] fix model conversion of 0.12 (#1941) Fix #1934. Add `descrpt_attr/rcut`, `descrpt_attr/ntypes`, `fitting_attr/dfparam`, `model_type`, `o_energy`, `o_force`, `o_virial`, `o_atom_energy`, `o_atom_virial`. Signed-off-by: Jinzhe Zeng --- deepmd/utils/convert.py | 228 +++++++++++++++++++++++++--------------- 1 file changed, 142 insertions(+), 86 deletions(-) diff --git a/deepmd/utils/convert.py b/deepmd/utils/convert.py index bbe9e067ef..6fbc9a36ac 100644 --- a/deepmd/utils/convert.py +++ b/deepmd/utils/convert.py @@ -1,4 +1,5 @@ import os +import textwrap from deepmd.env import tf from google.protobuf import text_format @@ -165,10 +166,65 @@ def convert_dp012_to_dp10(file: str): """ with open(file) as fp: file_content = fp.read() + # note: atom_energy must be put before energy, + # otherwise atom_energy_test -> atom_o_energy file_content = file_content\ .replace('DescrptNorot', 'DescrptSeA') \ .replace('ProdForceNorot', 'ProdForceSeA') \ - .replace('ProdVirialNorot', 'ProdVirialSeA') + .replace('ProdVirialNorot', 'ProdVirialSeA') \ + .replace('t_rcut', 'descrpt_attr/rcut') \ + .replace('t_ntypes', 'descrpt_attr/ntypes') \ + .replace('atom_energy_test', 'o_atom_energy') \ + .replace('atom_virial_test', 'o_atom_virial') \ + .replace('energy_test', 'o_energy') \ + .replace('force_test', 'o_force') \ + .replace('virial_test', 'o_virial') + file_content += textwrap.dedent("""\ + node { + name: "fitting_attr/dfparam" + op: "Const" + attr { + key: "dtype" + value { + type: DT_INT32 + } + } + attr { + key: "value" + value { + tensor { + dtype: DT_INT32 + tensor_shape { + } + int_val: 0 + } + } + } + } + """) + file_content += textwrap.dedent("""\ + node { + name: "model_attr/model_type" + op: "Const" + attr { + key: "dtype" + value { + type: DT_STRING + } + } + attr { + key: "value" + value { + tensor { + dtype: DT_STRING + tensor_shape { + } + string_val: "ener" + } + } + } + } + """) with open(file, 'w') as fp: fp.write(file_content) @@ -182,28 +238,28 @@ def convert_dp10_to_dp11(file: str): filename of the graph text """ with open(file, 'a') as f: - f.write(""" -node { - name: "fitting_attr/daparam" - op: "Const" - attr { - key: "dtype" - value { - type: DT_INT32 - } - } - attr { - key: "value" - value { - tensor { - dtype: DT_INT32 - tensor_shape { - } - int_val: 0 - } - } } -} -""") + f.write(textwrap.dedent("""\ + node { + name: "fitting_attr/daparam" + op: "Const" + attr { + key: "dtype" + value { + type: DT_INT32 + } + } + attr { + key: "value" + value { + tensor { + dtype: DT_INT32 + tensor_shape { + } + int_val: 0 + } + } } + } + """)) def convert_dp12_to_dp13(file: str): @@ -246,29 +302,29 @@ def convert_dp13_to_dp20(fname: str): """ with open(fname) as fp: file_content = fp.read() - file_content += """ -node { - name: "model_attr/model_version" - op: "Const" - attr { - key: "dtype" - value { - type: DT_STRING - } - } - attr { - key: "value" - value { - tensor { - dtype: DT_STRING - tensor_shape { + file_content += textwrap.dedent("""\ + node { + name: "model_attr/model_version" + op: "Const" + attr { + key: "dtype" + value { + type: DT_STRING + } + } + attr { + key: "value" + value { + tensor { + dtype: DT_STRING + tensor_shape { + } + string_val: "1.0" + } + } } - string_val: "1.0" } - } - } -} -""" + """) file_content = file_content\ .replace('DescrptSeA', 'ProdEnvMatA')\ .replace('DescrptSeR', 'ProdEnvMatR') @@ -278,52 +334,52 @@ def convert_dp13_to_dp20(fname: str): def convert_dp20_to_dp21(fname: str): with open(fname) as fp: file_content = fp.read() - old_model_version_node = """ -node { - name: "model_attr/model_version" - op: "Const" - attr { - key: "dtype" - value { - type: DT_STRING - } - } - attr { - key: "value" - value { - tensor { - dtype: DT_STRING - tensor_shape { + old_model_version_node = textwrap.dedent("""\ + node { + name: "model_attr/model_version" + op: "Const" + attr { + key: "dtype" + value { + type: DT_STRING + } + } + attr { + key: "value" + value { + tensor { + dtype: DT_STRING + tensor_shape { + } + string_val: "1.0" + } + } } - string_val: "1.0" } - } - } -} -""" - new_model_version_node = """ -node { - name: "model_attr/model_version" - op: "Const" - attr { - key: "dtype" - value { - type: DT_STRING - } - } - attr { - key: "value" - value { - tensor { - dtype: DT_STRING - tensor_shape { + """) + new_model_version_node = textwrap.dedent("""\ + node { + name: "model_attr/model_version" + op: "Const" + attr { + key: "dtype" + value { + type: DT_STRING + } + } + attr { + key: "value" + value { + tensor { + dtype: DT_STRING + tensor_shape { + } + string_val: "1.1" + } + } } - string_val: "1.1" } - } - } -} -""" + """) file_content = file_content\ .replace(old_model_version_node, new_model_version_node)\ .replace('TabulateFusion', 'TabulateFusionSeA')\ From ffb2fd8e706cd804b9dec602b533ac6257c7e702 Mon Sep 17 00:00:00 2001 From: Duo <50307526+iProzd@users.noreply.github.com> Date: Fri, 23 Sep 2022 15:21:20 +0800 Subject: [PATCH 21/21] fix bugs in type_idx_map (#1943) fix bugs in type_idx_map using np.searchsorted --- deepmd/utils/data.py | 3 ++- source/tests/test_data_large_batch.py | 3 ++- source/tests/water_se_atten_mixed_type.json | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/deepmd/utils/data.py b/deepmd/utils/data.py index 4ce3d43d91..e3dea5b04e 100644 --- a/deepmd/utils/data.py +++ b/deepmd/utils/data.py @@ -67,7 +67,8 @@ def __init__ (self, atom_type_ = [type_map.index(self.type_map[ii]) for ii in self.atom_type] self.atom_type = np.array(atom_type_, dtype = np.int32) else: - type_idx_map = np.searchsorted(type_map, self.type_map) + sorter = np.argsort(type_map) + type_idx_map = sorter[np.searchsorted(type_map, self.type_map, sorter=sorter)] try: atom_type_mix_ = np.array(type_idx_map)[self.atom_type_mix].astype(np.int32) except RuntimeError as e: diff --git a/source/tests/test_data_large_batch.py b/source/tests/test_data_large_batch.py index f84aaeedf6..e8522b5a70 100644 --- a/source/tests/test_data_large_batch.py +++ b/source/tests/test_data_large_batch.py @@ -33,8 +33,9 @@ def test_data_mixed_type(self): batch_size = 1 test_size = 1 rcut = j_must_have(jdata['model']['descriptor'], 'rcut') + type_map = j_must_have(jdata['model'], 'type_map') - data = DeepmdDataSystem(systems, batch_size, test_size, rcut) + data = DeepmdDataSystem(systems, batch_size, test_size, rcut, type_map=type_map) data_requirement = {'energy': {'ndof': 1, 'atomic': False, 'must': False, diff --git a/source/tests/water_se_atten_mixed_type.json b/source/tests/water_se_atten_mixed_type.json index e83e7c3f39..ac60aac921 100644 --- a/source/tests/water_se_atten_mixed_type.json +++ b/source/tests/water_se_atten_mixed_type.json @@ -1,7 +1,7 @@ { "_comment": " model parameters", "model" : { - "type_map": ["O", "H"], + "type_map": ["foo", "bar"], "type_embedding":{ "neuron": [8], "resnet_dt": false,