From 29b02ae476aea636aa95722bb28e72c30ae640a4 Mon Sep 17 00:00:00 2001 From: wei-kuochen Date: Wed, 20 Nov 2024 15:03:59 +0900 Subject: [PATCH 1/2] weko#46679 add parameters to tool --- modules/weko-records/weko_records/api.py | 28 ++++++++++++------------ scripts/demo/renew_all_item_types.py | 20 +++++++++++++++-- 2 files changed, 32 insertions(+), 16 deletions(-) diff --git a/modules/weko-records/weko_records/api.py b/modules/weko-records/weko_records/api.py index d9cd589dcf..1929d85b2b 100644 --- a/modules/weko-records/weko_records/api.py +++ b/modules/weko-records/weko_records/api.py @@ -915,11 +915,13 @@ def revisions(self): return RevisionsIterator(self.model) @classmethod - def reload(cls,itemtype_id): + def reload(cls, itemtype_id, specified_list=[], renew_value='None'): """reload itemtype properties. Args: itemtype_id (_type_): _description_ + specified_list: renew properties id list + renew_value: None, ALL, VAL, LOC """ # with db.session.begin_nested(): result = {"msg":"Update ItemType({})".format(itemtype_id),"code":0} @@ -935,6 +937,8 @@ def reload(cls,itemtype_id): multiple_flg = data['meta_list'][_prop_id]['option']['multiple'] if pat1.match(_tmp): _tmp = int(_tmp.replace('cus_', '')) + if specified_list and _tmp not in specified_list: + continue _prop = ItemTypeProps.get_record(_tmp) if _prop: # data['meta_list'][_prop_id] = json.loads('{"input_maxItems": "9999","input_minItems": "1","input_type": "cus_'+str(_prop.id)+'","input_value": "","option": {"crtf": false,"hidden": false,"multiple": true,"oneline": false,"required": false,"showlist": false},"title": "'+_prop.name+'","title_i18n": {"en": "", "ja": "'+_prop.name+'"}}') @@ -955,7 +959,7 @@ def reload(cls,itemtype_id): _forms = json.loads(json.dumps(pickle.loads(pickle.dumps(_prop.forms, -1))).replace('parentkey',_prop_id)) data['table_row_map']['form'][idx]=pickle.loads(pickle.dumps(_forms, -1)) _tmp_data = data['table_row_map']['form'][idx] - cls.update_attribute_options(tmp_data, _tmp_data) + cls.update_attribute_options(tmp_data, _tmp_data, renew_value) cls.update_property_enum(item_type.render['table_row_map']['schema']['properties'][_prop_id],data['table_row_map']['schema']['properties'][_prop_id]) else: tmp_data = pickle.loads(pickle.dumps(data['table_row_map']['form'][idx], -1)) @@ -964,7 +968,7 @@ def reload(cls,itemtype_id): _form = json.loads(json.dumps(pickle.loads(pickle.dumps(_prop.form, -1))).replace('parentkey',_prop_id)) data['table_row_map']['form'][idx]=pickle.loads(pickle.dumps(_form, -1)) _tmp_data = data['table_row_map']['form'][idx] - cls.update_attribute_options(tmp_data, _tmp_data) + cls.update_attribute_options(tmp_data, _tmp_data, renew_value) cls.update_property_enum(item_type.render['table_row_map']['schema']['properties'][_prop_id],data['table_row_map']['schema']['properties'][_prop_id]) from weko_itemtypes_ui.utils import fix_json_schema,update_required_schema_not_exist_in_form, update_text_and_textarea @@ -1028,7 +1032,7 @@ def update_property_enum(cls, old_value, new_value): cls.update_property_enum(old_value[key], new_value[key]) @classmethod - def update_attribute_options(cls, old_value, new_value): + def update_attribute_options(cls, old_value, new_value, renew_value): if "items" in old_value: for idx2,item2 in enumerate(old_value["items"]): isHide = False @@ -1049,12 +1053,14 @@ def update_attribute_options(cls, old_value, new_value): isSpecifyNewline = item2["isSpecifyNewline"] if "required" in item2: isRequired = item2["required"] - if "title_i18n" in item2: + if "title_i18n" in item2 and renew_value not in ["ALL", "LOC"]: title_i18n_temp = item2["title_i18n"] title_i18n = title_i18n_temp - if "title_i18n_temp" in item2: + if "title_i18n_temp" in item2 and renew_value not in ["ALL", "LOC"]: title_i18n_temp = item2["title_i18n_temp"] - if "titleMap" in item2: + if ("titleMap" in item2 and renew_value not in ["ALL", "VAL"]) or \ + ('titleMap' in new_value["items"][idx2] and + not new_value["items"][idx2]["titleMap"]): titleMap = item2["titleMap"] new_value["items"][idx2]["isHide"] = isHide @@ -1070,13 +1076,7 @@ def update_attribute_options(cls, old_value, new_value): new_value["items"][idx2]["titleMap"] = titleMap if 'items' in item2: - cls.update_attribute_options(item2, new_value["items"][idx2]) - - - - - - + cls.update_attribute_options(item2, new_value["items"][idx2], renew_value) class ItemTypeEditHistory(object): diff --git a/scripts/demo/renew_all_item_types.py b/scripts/demo/renew_all_item_types.py index 12a086a5cf..19e5f11011 100644 --- a/scripts/demo/renew_all_item_types.py +++ b/scripts/demo/renew_all_item_types.py @@ -1,13 +1,22 @@ from invenio_db import db from weko_records.api import ItemTypes +from properties import property_config + import traceback +import argparse def main(): try: - + _renew_type = args.renew_type if args else '' + _renew_value = args.renew_value if args else 'None' + specified_list = property_config.SPECIFIED_LIST if _renew_type == 'only_specified' else [] itemtypes = ItemTypes.get_all() + print("Process property id list:{}".format(specified_list)) for itemtype in itemtypes: - ret = ItemTypes.reload(itemtype.id) + if _renew_type == 'only_specified': + ret = ItemTypes.reload(itemtype.id, specified_list, _renew_value) + else: + ret = ItemTypes.reload(itemtype.id) print("itemtype id:{}, itemtype name:{}".format(itemtype.id,itemtype.item_type_name.name)) print(ret['msg']) db.session.commit() @@ -18,4 +27,11 @@ def main(): if __name__ == '__main__': + try: + parser = argparse.ArgumentParser() + parser.add_argument('renew_type', action='store') + parser.add_argument('renew_value', action='store') + args = parser.parse_args() + except: + args = None main() \ No newline at end of file From 36185cf7f8a714078a08d02603c31ac87a9135b2 Mon Sep 17 00:00:00 2001 From: wei-kuochen Date: Thu, 12 Dec 2024 17:29:43 +0900 Subject: [PATCH 2/2] weko#46668 fix type change issue --- .../invenio_oaiharvester/tasks.py | 6 ++ modules/weko-deposit/weko_deposit/api.py | 2 + modules/weko-records/weko_records/utils.py | 56 +++++++++++-------- 3 files changed, 42 insertions(+), 22 deletions(-) diff --git a/modules/invenio-oaiharvester/invenio_oaiharvester/tasks.py b/modules/invenio-oaiharvester/invenio_oaiharvester/tasks.py index dbc8d21794..1c3e5d0df5 100644 --- a/modules/invenio-oaiharvester/invenio_oaiharvester/tasks.py +++ b/modules/invenio-oaiharvester/invenio_oaiharvester/tasks.py @@ -445,6 +445,7 @@ def sigterm_handler(*args): nonlocal pause pause = True signal.signal(signal.SIGTERM, sigterm_handler) + _errormsg = [] while True: records, rtoken = harvester_list_records( harvesting.base_url, @@ -459,6 +460,9 @@ def sigterm_handler(*args): try: process_item(record, harvesting, counter, request_info) db.session.commit() + except ValueError as ex: + _errormsg.append(str(ex)) + event_counter('error_items', counter) except Exception as ex: current_app.logger.debug(traceback.format_exc()) current_app.logger.error( @@ -473,6 +477,8 @@ def sigterm_handler(*args): elif pause is True: harvest_log.status = 'Suspended' break + if _errormsg: + harvest_log.errmsg = '\n'.join(_errormsg)[:255] except Exception as ex: db.session.rollback() harvest_log.status = 'Failed' diff --git a/modules/weko-deposit/weko_deposit/api.py b/modules/weko-deposit/weko_deposit/api.py index e0579f6905..54f6d807f5 100644 --- a/modules/weko-deposit/weko_deposit/api.py +++ b/modules/weko-deposit/weko_deposit/api.py @@ -1439,6 +1439,8 @@ def convert_item_metadata(self, index_obj, data=None): self._convert_jpcoar_data_to_es() except RuntimeError: raise + except ValueError as ex: + raise ex except BaseException: import traceback current_app.logger.error(traceback.format_exc()) diff --git a/modules/weko-records/weko_records/utils.py b/modules/weko-records/weko_records/utils.py index 0fbe4315dd..a5dfcca403 100644 --- a/modules/weko-records/weko_records/utils.py +++ b/modules/weko-records/weko_records/utils.py @@ -312,7 +312,15 @@ def copy_field_test(dc, map, jrc, iid=None): b = None if idx < len(_lte): b = _lte[idx] - ranges.append(convert_range_value(a, b)) + try: + ranges.append(convert_range_value(a, b)) + except: + _error_col = val.get("path", {}).get("gte") \ + if val.get("path", {}).get("gte") else val.get("path", {}).get("lte") + raise ValueError( + "can not convert to range value. start:{} end:{}. column: {}".format( + a, b, _error_col) + ) if len(ranges) > 0: value_range = {id: ranges} jrc.update(value_range) @@ -331,7 +339,15 @@ def copy_field_test(dc, map, jrc, iid=None): b = None if idx < len(_lte): b = _lte[idx] - dateRanges.append(convert_date_range_value(a, b)) + try: + dateRanges.append(convert_date_range_value(a, b)) + except: + _error_col = val.get("path", {}).get("gte") \ + if val.get("path", {}).get("gte") else val.get("path", {}).get("lte") + raise ValueError( + "can not convert to range value. start:{} end:{}. column: {}".format( + a, b, _error_col) + ) if len(dateRanges) > 0: value_range = {id: dateRanges} jrc.update(value_range) @@ -403,20 +419,13 @@ def convert_range_value(start, end=None): else: ret = {_start: end, _end: start} else: - try: - a = float(start) - b = float(end) + a = float(start) + b = float(end) - if a < b: - ret = {_start: start, _end: end} - else: - ret = {_start: end, _end: start} - except ValueError: - current_app.logger.exception( - "can not convert to range value. start:{0} end:{1}".format( - start, end - ) - ) + if a < b: + ret = {_start: start, _end: end} + else: + ret = {_start: end, _end: start} return ret @@ -479,15 +488,18 @@ def makeDateRangeValue(start, end): a = None b = None if p2.match(start): - a = time.strptime(start, "%Y-%m-%d") - b = time.strptime(end, "%Y-%m-%d") - + _s = start.split('-') + _e = end.split('-') + a = time.strptime('{:0>4}-{}-{}'.format(_s[0], _s[1], _s[2]), "%Y-%m-%d") + b = time.strptime('{:0>4}-{}-{}'.format(_e[0], _e[1], _e[2]), "%Y-%m-%d") elif p3.match(start): - a = time.strptime(start, "%Y-%m") - b = time.strptime(end, "%Y-%m") + _s = start.split('-') + _e = end.split('-') + a = time.strptime('{:0>4}-{}'.format(_s[0], _s[1]), "%Y-%m") + b = time.strptime('{:0>4}-{}'.format(_e[0], _e[1]), "%Y-%m") elif p4.match(start): - a = time.strptime(start, "%Y") - b = time.strptime(end, "%Y") + a = time.strptime('{:0>4}'.format(start), "%Y") + b = time.strptime('{:0>4}'.format(end), "%Y") if a is not None and b is not None: if a < b: