From 8b8a344451fe30ebd06b2fb8edbc9da93745fe9f Mon Sep 17 00:00:00 2001 From: tejaskriya Date: Mon, 22 Jul 2024 11:26:20 +0530 Subject: [PATCH 1/6] HDDS-11190. Add --fields option to ldb scan command --- .../apache/hadoop/ozone/debug/DBScanner.java | 126 +++++++++++++++++- .../hadoop/ozone/debug/ValueSchema.java | 12 +- 2 files changed, 128 insertions(+), 10 deletions(-) diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/DBScanner.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/DBScanner.java index 0c38fbe33ba1..b7dd33508559 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/DBScanner.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/DBScanner.java @@ -55,9 +55,12 @@ import java.io.BufferedWriter; import java.io.IOException; import java.io.PrintWriter; +import java.lang.reflect.Field; +import java.lang.reflect.ParameterizedType; import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -121,6 +124,11 @@ public class DBScanner implements Callable, SubcommandWithParent { description = "Key at which iteration of the DB ends") private String endKey; + @CommandLine.Option(names = {"--fields"}, + description = "Comma-separated list of fields needed for each value. " + + "eg.) \"name,acls.type\" for showing name and type under acls.") + private String fieldsFilter; + @CommandLine.Option(names = {"--dnSchema", "--dn-schema", "-d"}, description = "Datanode DB Schema Version: V1/V2/V3", defaultValue = "V3") @@ -291,7 +299,7 @@ private void processRecords(ManagedRocksIterator iterator, } Future future = threadPool.submit( new Task(dbColumnFamilyDef, batch, logWriter, sequenceId, - withKey, schemaV3)); + withKey, schemaV3, fieldsFilter)); futures.add(future); batch = new ArrayList<>(batchSize); sequenceId++; @@ -299,7 +307,7 @@ private void processRecords(ManagedRocksIterator iterator, } if (!batch.isEmpty()) { Future future = threadPool.submit(new Task(dbColumnFamilyDef, - batch, logWriter, sequenceId, withKey, schemaV3)); + batch, logWriter, sequenceId, withKey, schemaV3, fieldsFilter)); futures.add(future); } @@ -378,6 +386,12 @@ private boolean printTable(List columnFamilyHandleList, return true; } + // check if fields are valid + if (fieldsFilter != null && + !checkValidValueFields(dbPath, fieldsFilter, columnFamilyDefinition)){ + return false; + } + ManagedRocksIterator iterator = null; ManagedReadOptions readOptions = null; ManagedSlice slice = null; @@ -405,6 +419,46 @@ private boolean printTable(List columnFamilyHandleList, IOUtils.closeQuietly(iterator, readOptions, slice); } } + boolean checkValidValueFields(String dbPath, String valueFields, DBColumnFamilyDefinition columnFamilyDefinition) { + Map valueSchema = new HashMap<>(); + try { + boolean schemaSuccess = new ValueSchema().getValueFields(dbPath, valueSchema, 2, tableName); + if (!schemaSuccess) { + err().println("Error: Schema for table with name '" + tableName + "' not found"); + return false; + } + Map valueClassSchema = + (Map) valueSchema.get(columnFamilyDefinition.getValueType().getSimpleName()); + + if (valueClassSchema == null) { + err().print("Error: Schema for table with name '" + tableName + + "' and value type '" + columnFamilyDefinition.getValueType().getSimpleName() + "' not found"); + return false; + } + + for (String field : valueFields.split(",")) { // TODO: check for distinct values + String[] subfields = field.split("\\."); + if (subfields.length > 2) { + err().println("Warn: Fields only up to 2nd level is allowed. Ignoring the subfields from 3rd level onwards"); + } + Object subfieldValueSchema = valueClassSchema.get(subfields[0]); + if (subfieldValueSchema == null) { + err().println("Error: Field with name '" + field + "' not found"); + return false; + } + if ((subfields.length > 1) && + (subfieldValueSchema instanceof String || + ((Map) subfieldValueSchema).get(subfields[1]) == null)) { + err().println("Error: subField with name '" + field + "' not found"); + return false; + } + } + } catch (Exception ex) { + err().println("exception from fields check:" + ex.getMessage()); + } + + return true; + } private String removeTrailingSlashIfNeeded(String dbPath) { if (dbPath.endsWith(OzoneConsts.OZONE_URI_DELIMITER)) { @@ -465,16 +519,18 @@ private static class Task implements Callable { private final long sequenceId; private final boolean withKey; private final boolean schemaV3; + private String valueFields; Task(DBColumnFamilyDefinition dbColumnFamilyDefinition, ArrayList batch, LogWriter logWriter, - long sequenceId, boolean withKey, boolean schemaV3) { + long sequenceId, boolean withKey, boolean schemaV3, String valueFields) { this.dbColumnFamilyDefinition = dbColumnFamilyDefinition; this.batch = batch; this.logWriter = logWriter; this.sequenceId = sequenceId; this.withKey = withKey; this.schemaV3 = schemaV3; + this.valueFields = valueFields; } @Override @@ -515,7 +571,53 @@ public Void call() { Object o = dbColumnFamilyDefinition.getValueCodec() .fromPersistedFormat(byteArrayKeyValue.getValue()); - sb.append(WRITER.writeValueAsString(o)); + + if (valueFields != null) { + Map filteredValue = new HashMap<>(); + for (String field : valueFields.split(",")) { + String[] subfields = field.split("\\."); + try { + Field valueClassField = getRequiredFieldFromAllFields(dbColumnFamilyDefinition.getValueType(), subfields[0]); + Object valueObject = valueClassField.get(o); + if (subfields.length == 1) { + filteredValue.put(field, valueObject); + } else { + Class typeOfClassField; + try { + if (Collection.class.isAssignableFrom(valueClassField.getType())) { + typeOfClassField = (Class) + ((ParameterizedType) valueClassField.getGenericType()).getActualTypeArguments()[0]; + } else { + typeOfClassField = valueClassField.getType(); + } + } catch (ClassCastException ex) { + typeOfClassField = valueClassField.getType(); + } + + Field classSubField = getRequiredFieldFromAllFields(typeOfClassField, subfields[1]); + + if (Collection.class.isAssignableFrom(valueObject.getClass())) { + List> subfieldObjectsList = new ArrayList<>(); + for (Object ob : (List) valueObject) { + Map subfieldValueMap = new HashMap<>(); + subfieldValueMap.put(subfields[1],classSubField.get(ob)); + subfieldObjectsList.add(subfieldValueMap); + } + filteredValue.put(field, subfieldObjectsList); + } else { + filteredValue.put(field, classSubField.get(valueObject)); + } + } + } catch (NoSuchFieldException ex) { + err().println("ERROR: no such field: " + valueFields); + } + } + sb.append(WRITER.writeValueAsString(filteredValue)); + + } else { + sb.append(WRITER.writeValueAsString(o)); + } + results.add(sb.toString()); } logWriter.log(results, sequenceId); @@ -525,6 +627,22 @@ public Void call() { } return null; } + + Field getRequiredFieldFromAllFields(Class clazz, String fieldName) throws NoSuchFieldException { + List classFieldList = new ValueSchema().getAllFields(clazz); + Field classField = null; + for (Field f : classFieldList) { + if (f.getName().equals(fieldName)) { + classField = f; + break; + } + } + if (classField == null) { + throw new NoSuchFieldException(); + } + classField.setAccessible(true); + return classField; + } } private static class ByteArrayKeyValue { diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/ValueSchema.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/ValueSchema.java index a5029b3e6b90..7212d4efa3b8 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/ValueSchema.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/ValueSchema.java @@ -88,7 +88,7 @@ public Void call() throws Exception { String dbPath = parent.getDbPath(); Map fields = new HashMap<>(); - success = getValueFields(dbPath, fields); + success = getValueFields(dbPath, fields, depth, tableName); out().println(JsonUtils.toJsonStringWithDefaultPrettyPrinter(fields)); @@ -101,7 +101,7 @@ public Void call() throws Exception { return null; } - private boolean getValueFields(String dbPath, Map valueSchema) { + public boolean getValueFields(String dbPath, Map valueSchema, int d, String table) { dbPath = removeTrailingSlashIfNeeded(dbPath); DBDefinitionFactory.setDnDBSchemaVersion(dnDBSchemaVersion); @@ -111,14 +111,14 @@ private boolean getValueFields(String dbPath, Map valueSchema) { return false; } final DBColumnFamilyDefinition columnFamilyDefinition = - dbDefinition.getColumnFamily(tableName); + dbDefinition.getColumnFamily(table); if (columnFamilyDefinition == null) { - err().print("Error: Table with name '" + tableName + "' not found"); + err().print("Error: Table with name '" + table + "' not found"); return false; } Class c = columnFamilyDefinition.getValueType(); - valueSchema.put(c.getSimpleName(), getFieldsStructure(c, depth)); + valueSchema.put(c.getSimpleName(), getFieldsStructure(c, d)); return true; } @@ -148,7 +148,7 @@ private Object getFieldsStructure(Class clazz, int currentDepth) { } } - private List getAllFields(Class clazz) { + public List getAllFields(Class clazz) { // NOTE: Schema of interface type, like ReplicationConfig, cannot be fetched. // An empty list "[]" will be shown for such types of fields. if (clazz == null) { From 309debe7ca7d11c86bf341ecc22375dada4a1fc6 Mon Sep 17 00:00:00 2001 From: tejaskriya Date: Mon, 22 Jul 2024 11:49:35 +0530 Subject: [PATCH 2/6] checkstyle fix --- .../org/apache/hadoop/ozone/debug/DBScanner.java | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/DBScanner.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/DBScanner.java index b7dd33508559..47ddedbcf161 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/DBScanner.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/DBScanner.java @@ -388,7 +388,7 @@ private boolean printTable(List columnFamilyHandleList, // check if fields are valid if (fieldsFilter != null && - !checkValidValueFields(dbPath, fieldsFilter, columnFamilyDefinition)){ + !checkValidValueFields(dbPath, fieldsFilter, columnFamilyDefinition)) { return false; } @@ -419,7 +419,8 @@ private boolean printTable(List columnFamilyHandleList, IOUtils.closeQuietly(iterator, readOptions, slice); } } - boolean checkValidValueFields(String dbPath, String valueFields, DBColumnFamilyDefinition columnFamilyDefinition) { + boolean checkValidValueFields(String dbPath, String valueFields, + DBColumnFamilyDefinition columnFamilyDefinition) { Map valueSchema = new HashMap<>(); try { boolean schemaSuccess = new ValueSchema().getValueFields(dbPath, valueSchema, 2, tableName); @@ -577,7 +578,8 @@ public Void call() { for (String field : valueFields.split(",")) { String[] subfields = field.split("\\."); try { - Field valueClassField = getRequiredFieldFromAllFields(dbColumnFamilyDefinition.getValueType(), subfields[0]); + Field valueClassField = getRequiredFieldFromAllFields(dbColumnFamilyDefinition.getValueType(), + subfields[0]); Object valueObject = valueClassField.get(o); if (subfields.length == 1) { filteredValue.put(field, valueObject); @@ -597,10 +599,10 @@ public Void call() { Field classSubField = getRequiredFieldFromAllFields(typeOfClassField, subfields[1]); if (Collection.class.isAssignableFrom(valueObject.getClass())) { - List> subfieldObjectsList = new ArrayList<>(); + List> subfieldObjectsList = new ArrayList<>(); for (Object ob : (List) valueObject) { Map subfieldValueMap = new HashMap<>(); - subfieldValueMap.put(subfields[1],classSubField.get(ob)); + subfieldValueMap.put(subfields[1], classSubField.get(ob)); subfieldObjectsList.add(subfieldValueMap); } filteredValue.put(field, subfieldObjectsList); @@ -609,7 +611,7 @@ public Void call() { } } } catch (NoSuchFieldException ex) { - err().println("ERROR: no such field: " + valueFields); + err().println("ERROR: no such field: " + valueFields); } } sb.append(WRITER.writeValueAsString(filteredValue)); From a43e85368f7bdaef71d29ca988d7a12990cef3ad Mon Sep 17 00:00:00 2001 From: tejaskriya Date: Mon, 22 Jul 2024 12:16:03 +0530 Subject: [PATCH 3/6] findbugs fix --- .../src/main/java/org/apache/hadoop/ozone/debug/DBScanner.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/DBScanner.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/DBScanner.java index 47ddedbcf161..c504eeada190 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/DBScanner.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/DBScanner.java @@ -623,7 +623,7 @@ public Void call() { results.add(sb.toString()); } logWriter.log(results, sequenceId); - } catch (Exception e) { + } catch (IOException | IllegalAccessException e) { exception = true; LOG.error("Exception parse Object", e); } From 64ee30bf578ab23278279f1254f47ba2ea0e8aca Mon Sep 17 00:00:00 2001 From: tejaskriya Date: Tue, 20 Aug 2024 11:56:08 +0530 Subject: [PATCH 4/6] Recursive filtering of object --- .../apache/hadoop/ozone/debug/DBScanner.java | 136 +++++++----------- .../hadoop/ozone/debug/ValueSchema.java | 11 +- 2 files changed, 57 insertions(+), 90 deletions(-) diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/DBScanner.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/DBScanner.java index c504eeada190..24e687b27648 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/DBScanner.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/DBScanner.java @@ -386,12 +386,6 @@ private boolean printTable(List columnFamilyHandleList, return true; } - // check if fields are valid - if (fieldsFilter != null && - !checkValidValueFields(dbPath, fieldsFilter, columnFamilyDefinition)) { - return false; - } - ManagedRocksIterator iterator = null; ManagedReadOptions readOptions = null; ManagedSlice slice = null; @@ -419,47 +413,6 @@ private boolean printTable(List columnFamilyHandleList, IOUtils.closeQuietly(iterator, readOptions, slice); } } - boolean checkValidValueFields(String dbPath, String valueFields, - DBColumnFamilyDefinition columnFamilyDefinition) { - Map valueSchema = new HashMap<>(); - try { - boolean schemaSuccess = new ValueSchema().getValueFields(dbPath, valueSchema, 2, tableName); - if (!schemaSuccess) { - err().println("Error: Schema for table with name '" + tableName + "' not found"); - return false; - } - Map valueClassSchema = - (Map) valueSchema.get(columnFamilyDefinition.getValueType().getSimpleName()); - - if (valueClassSchema == null) { - err().print("Error: Schema for table with name '" + tableName + - "' and value type '" + columnFamilyDefinition.getValueType().getSimpleName() + "' not found"); - return false; - } - - for (String field : valueFields.split(",")) { // TODO: check for distinct values - String[] subfields = field.split("\\."); - if (subfields.length > 2) { - err().println("Warn: Fields only up to 2nd level is allowed. Ignoring the subfields from 3rd level onwards"); - } - Object subfieldValueSchema = valueClassSchema.get(subfields[0]); - if (subfieldValueSchema == null) { - err().println("Error: Field with name '" + field + "' not found"); - return false; - } - if ((subfields.length > 1) && - (subfieldValueSchema instanceof String || - ((Map) subfieldValueSchema).get(subfields[1]) == null)) { - err().println("Error: subField with name '" + field + "' not found"); - return false; - } - } - } catch (Exception ex) { - err().println("exception from fields check:" + ex.getMessage()); - } - - return true; - } private String removeTrailingSlashIfNeeded(String dbPath) { if (dbPath.endsWith(OzoneConsts.OZONE_URI_DELIMITER)) { @@ -538,6 +491,14 @@ private static class Task implements Callable { public Void call() { try { ArrayList results = new ArrayList<>(batch.size()); + List> fieldsSplit = new ArrayList<>(); + if (valueFields != null) { + for (String field : valueFields.split(",")) { + String[] subfields = field.split("\\."); + fieldsSplit.add(Arrays.asList(subfields)); + } + } + for (ByteArrayKeyValue byteArrayKeyValue : batch) { StringBuilder sb = new StringBuilder(); if (!(sequenceId == FIRST_SEQUENCE_ID && results.isEmpty())) { @@ -575,47 +536,16 @@ public Void call() { if (valueFields != null) { Map filteredValue = new HashMap<>(); - for (String field : valueFields.split(",")) { - String[] subfields = field.split("\\."); + for (List subfields : fieldsSplit) { try { - Field valueClassField = getRequiredFieldFromAllFields(dbColumnFamilyDefinition.getValueType(), - subfields[0]); - Object valueObject = valueClassField.get(o); - if (subfields.length == 1) { - filteredValue.put(field, valueObject); - } else { - Class typeOfClassField; - try { - if (Collection.class.isAssignableFrom(valueClassField.getType())) { - typeOfClassField = (Class) - ((ParameterizedType) valueClassField.getGenericType()).getActualTypeArguments()[0]; - } else { - typeOfClassField = valueClassField.getType(); - } - } catch (ClassCastException ex) { - typeOfClassField = valueClassField.getType(); - } - - Field classSubField = getRequiredFieldFromAllFields(typeOfClassField, subfields[1]); - - if (Collection.class.isAssignableFrom(valueObject.getClass())) { - List> subfieldObjectsList = new ArrayList<>(); - for (Object ob : (List) valueObject) { - Map subfieldValueMap = new HashMap<>(); - subfieldValueMap.put(subfields[1], classSubField.get(ob)); - subfieldObjectsList.add(subfieldValueMap); - } - filteredValue.put(field, subfieldObjectsList); - } else { - filteredValue.put(field, classSubField.get(valueObject)); - } - } + filteredValue.putAll(getFilteredObject(o, subfields, dbColumnFamilyDefinition.getValueType())); } catch (NoSuchFieldException ex) { - err().println("ERROR: no such field: " + valueFields); + err().println("ERROR: no such field: " + subfields); + } catch (IllegalAccessException e) { + err().println("ERROR: Cannot get field from object: " + subfields); } } sb.append(WRITER.writeValueAsString(filteredValue)); - } else { sb.append(WRITER.writeValueAsString(o)); } @@ -623,15 +553,51 @@ public Void call() { results.add(sb.toString()); } logWriter.log(results, sequenceId); - } catch (IOException | IllegalAccessException e) { + } catch (IOException e) { exception = true; LOG.error("Exception parse Object", e); } return null; } + Map getFilteredObject(Object o, List subfields, Class fieldClass) + throws NoSuchFieldException, IllegalAccessException { + Field valueClassField = getRequiredFieldFromAllFields(fieldClass, subfields.get(0)); + Object valueObject = valueClassField.get(o); + int length = subfields.size(); + Map valueMap = new HashMap<>(); + if (length == 1) { + valueMap.put(subfields.get(0), valueObject); + } else { + Class typeOfClassField; + try { + if (Collection.class.isAssignableFrom(valueClassField.getType())) { + typeOfClassField = (Class) + ((ParameterizedType) valueClassField.getGenericType()).getActualTypeArguments()[0]; + } else { + typeOfClassField = valueClassField.getType(); + } + } catch (ClassCastException ex) { + typeOfClassField = valueClassField.getType(); + } + + if (Collection.class.isAssignableFrom(valueObject.getClass())) { + List subfieldObjectsList = new ArrayList<>(); + for (Object ob : (List) valueObject) { + Object subfieldValue = getFilteredObject(ob, subfields.subList(1, length), typeOfClassField); + subfieldObjectsList.add(subfieldValue); + } + valueMap.put(subfields.get(0), subfieldObjectsList); + } else { + valueMap.put(subfields.get(0), + getFilteredObject(valueObject, subfields.subList(1, length), typeOfClassField)); + } + } + return valueMap; + } + Field getRequiredFieldFromAllFields(Class clazz, String fieldName) throws NoSuchFieldException { - List classFieldList = new ValueSchema().getAllFields(clazz); + List classFieldList = ValueSchema.getAllFields(clazz); Field classField = null; for (Field f : classFieldList) { if (f.getName().equals(fieldName)) { diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/ValueSchema.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/ValueSchema.java index 7212d4efa3b8..b06be2aff534 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/ValueSchema.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/ValueSchema.java @@ -88,7 +88,7 @@ public Void call() throws Exception { String dbPath = parent.getDbPath(); Map fields = new HashMap<>(); - success = getValueFields(dbPath, fields, depth, tableName); + success = getValueFields(dbPath, fields, depth, tableName, dnDBSchemaVersion); out().println(JsonUtils.toJsonStringWithDefaultPrettyPrinter(fields)); @@ -101,7 +101,8 @@ public Void call() throws Exception { return null; } - public boolean getValueFields(String dbPath, Map valueSchema, int d, String table) { + public static boolean getValueFields(String dbPath, Map valueSchema, int d, String table, + String dnDBSchemaVersion) { dbPath = removeTrailingSlashIfNeeded(dbPath); DBDefinitionFactory.setDnDBSchemaVersion(dnDBSchemaVersion); @@ -123,7 +124,7 @@ public boolean getValueFields(String dbPath, Map valueSchema, in return true; } - private Object getFieldsStructure(Class clazz, int currentDepth) { + private static Object getFieldsStructure(Class clazz, int currentDepth) { if (clazz.isPrimitive() || String.class.equals(clazz)) { return clazz.getSimpleName(); } else if (currentDepth == 0) { @@ -148,7 +149,7 @@ private Object getFieldsStructure(Class clazz, int currentDepth) { } } - public List getAllFields(Class clazz) { + public static List getAllFields(Class clazz) { // NOTE: Schema of interface type, like ReplicationConfig, cannot be fetched. // An empty list "[]" will be shown for such types of fields. if (clazz == null) { @@ -176,7 +177,7 @@ public Class getParentType() { return RDBParser.class; } - private String removeTrailingSlashIfNeeded(String dbPath) { + private static String removeTrailingSlashIfNeeded(String dbPath) { if (dbPath.endsWith(OzoneConsts.OZONE_URI_DELIMITER)) { dbPath = dbPath.substring(0, dbPath.length() - 1); } From 4ebaa601d4112620f0dfc06e55d37a47699a2b35 Mon Sep 17 00:00:00 2001 From: tejaskriya Date: Fri, 23 Aug 2024 12:30:45 +0530 Subject: [PATCH 5/6] Handle map values and remove redundant code --- .../apache/hadoop/ozone/debug/DBScanner.java | 60 +++++++++++-------- 1 file changed, 34 insertions(+), 26 deletions(-) diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/DBScanner.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/DBScanner.java index 24e687b27648..b29fadcda5a3 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/DBScanner.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/DBScanner.java @@ -56,7 +56,6 @@ import java.io.IOException; import java.io.PrintWriter; import java.lang.reflect.Field; -import java.lang.reflect.ParameterizedType; import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; @@ -538,7 +537,7 @@ public Void call() { Map filteredValue = new HashMap<>(); for (List subfields : fieldsSplit) { try { - filteredValue.putAll(getFilteredObject(o, subfields, dbColumnFamilyDefinition.getValueType())); + filteredValue.putAll(getFilteredObject(o, dbColumnFamilyDefinition.getValueType(), subfields)); } catch (NoSuchFieldException ex) { err().println("ERROR: no such field: " + subfields); } catch (IllegalAccessException e) { @@ -560,42 +559,51 @@ public Void call() { return null; } - Map getFilteredObject(Object o, List subfields, Class fieldClass) + Map getFilteredObject(Object obj, Class clazz, List fields) throws NoSuchFieldException, IllegalAccessException { - Field valueClassField = getRequiredFieldFromAllFields(fieldClass, subfields.get(0)); - Object valueObject = valueClassField.get(o); - int length = subfields.size(); + Field valueClassField = getRequiredFieldFromAllFields(clazz, fields.get(0)); + Object valueObject = valueClassField.get(obj); + int length = fields.size(); Map valueMap = new HashMap<>(); + if (length == 1) { - valueMap.put(subfields.get(0), valueObject); + valueMap.put(fields.get(0), valueObject); } else { - Class typeOfClassField; - try { - if (Collection.class.isAssignableFrom(valueClassField.getType())) { - typeOfClassField = (Class) - ((ParameterizedType) valueClassField.getGenericType()).getActualTypeArguments()[0]; - } else { - typeOfClassField = valueClassField.getType(); - } - } catch (ClassCastException ex) { - typeOfClassField = valueClassField.getType(); - } - if (Collection.class.isAssignableFrom(valueObject.getClass())) { - List subfieldObjectsList = new ArrayList<>(); - for (Object ob : (List) valueObject) { - Object subfieldValue = getFilteredObject(ob, subfields.subList(1, length), typeOfClassField); - subfieldObjectsList.add(subfieldValue); + List subfieldObjectsList = + getFilteredObjectCollection((Collection) valueObject, fields.subList(1, length)); + valueMap.put(fields.get(0), subfieldObjectsList); + } else if (Map.class.isAssignableFrom(valueObject.getClass())) { + Map subfieldObjectsMap = new HashMap<>(); + Map valueObjectMap = (Map) valueObject; + for (Map.Entry ob : valueObjectMap.entrySet()) { + Object subfieldValue; + if (Collection.class.isAssignableFrom(ob.getValue().getClass())) { + subfieldValue = getFilteredObjectCollection((Collection)ob.getValue(), fields.subList(1, length)); + } else { + subfieldValue = getFilteredObject(ob.getValue(), ob.getValue().getClass(), fields.subList(1, length)); + } + subfieldObjectsMap.put(ob.getKey(), subfieldValue); } - valueMap.put(subfields.get(0), subfieldObjectsList); + valueMap.put(fields.get(0), subfieldObjectsMap); } else { - valueMap.put(subfields.get(0), - getFilteredObject(valueObject, subfields.subList(1, length), typeOfClassField)); + valueMap.put(fields.get(0), + getFilteredObject(valueObject, valueClassField.getType(), fields.subList(1, length))); } } return valueMap; } + List getFilteredObjectCollection(Collection valueObject, List fields) + throws NoSuchFieldException, IllegalAccessException { + List subfieldObjectsList = new ArrayList<>(); + for (Object ob : valueObject) { + Object subfieldValue = getFilteredObject(ob, ob.getClass(), fields); + subfieldObjectsList.add(subfieldValue); + } + return subfieldObjectsList; + } + Field getRequiredFieldFromAllFields(Class clazz, String fieldName) throws NoSuchFieldException { List classFieldList = ValueSchema.getAllFields(clazz); Field classField = null; From 94a579504b71ad4869fe9ac195cadd66deabb4ac Mon Sep 17 00:00:00 2001 From: tejaskriya Date: Mon, 26 Aug 2024 23:08:02 +0530 Subject: [PATCH 6/6] use map instead of list for fields tracking --- .../apache/hadoop/ozone/debug/DBScanner.java | 92 +++++++++++-------- 1 file changed, 55 insertions(+), 37 deletions(-) diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/DBScanner.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/DBScanner.java index b29fadcda5a3..4653aa3eeb31 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/DBScanner.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/DBScanner.java @@ -486,15 +486,34 @@ private static class Task implements Callable { this.valueFields = valueFields; } + Map getFieldSplit(List fields, Map fieldMap) { + int len = fields.size(); + if (fieldMap == null) { + fieldMap = new HashMap<>(); + } + if (len == 1) { + fieldMap.putIfAbsent(fields.get(0), null); + } else { + Map fieldMapGet = (Map) fieldMap.get(fields.get(0)); + if (fieldMapGet == null) { + fieldMap.put(fields.get(0), getFieldSplit(fields.subList(1, len), null)); + } else { + fieldMap.put(fields.get(0), getFieldSplit(fields.subList(1, len), fieldMapGet)); + } + } + return fieldMap; + } + @Override public Void call() { try { ArrayList results = new ArrayList<>(batch.size()); - List> fieldsSplit = new ArrayList<>(); + Map fieldsSplitMap = new HashMap<>(); + if (valueFields != null) { for (String field : valueFields.split(",")) { String[] subfields = field.split("\\."); - fieldsSplit.add(Arrays.asList(subfields)); + fieldsSplitMap = getFieldSplit(Arrays.asList(subfields), fieldsSplitMap); } } @@ -535,15 +554,7 @@ public Void call() { if (valueFields != null) { Map filteredValue = new HashMap<>(); - for (List subfields : fieldsSplit) { - try { - filteredValue.putAll(getFilteredObject(o, dbColumnFamilyDefinition.getValueType(), subfields)); - } catch (NoSuchFieldException ex) { - err().println("ERROR: no such field: " + subfields); - } catch (IllegalAccessException e) { - err().println("ERROR: Cannot get field from object: " + subfields); - } - } + filteredValue.putAll(getFilteredObject(o, dbColumnFamilyDefinition.getValueType(), fieldsSplitMap)); sb.append(WRITER.writeValueAsString(filteredValue)); } else { sb.append(WRITER.writeValueAsString(o)); @@ -559,42 +570,49 @@ public Void call() { return null; } - Map getFilteredObject(Object obj, Class clazz, List fields) - throws NoSuchFieldException, IllegalAccessException { - Field valueClassField = getRequiredFieldFromAllFields(clazz, fields.get(0)); - Object valueObject = valueClassField.get(obj); - int length = fields.size(); + Map getFilteredObject(Object obj, Class clazz, Map fieldsSplitMap) { Map valueMap = new HashMap<>(); + for (Map.Entry field : fieldsSplitMap.entrySet()) { + try { + Field valueClassField = getRequiredFieldFromAllFields(clazz, field.getKey()); + Object valueObject = valueClassField.get(obj); + Map subfields = (Map) field.getValue(); - if (length == 1) { - valueMap.put(fields.get(0), valueObject); - } else { - if (Collection.class.isAssignableFrom(valueObject.getClass())) { - List subfieldObjectsList = - getFilteredObjectCollection((Collection) valueObject, fields.subList(1, length)); - valueMap.put(fields.get(0), subfieldObjectsList); - } else if (Map.class.isAssignableFrom(valueObject.getClass())) { - Map subfieldObjectsMap = new HashMap<>(); - Map valueObjectMap = (Map) valueObject; - for (Map.Entry ob : valueObjectMap.entrySet()) { - Object subfieldValue; - if (Collection.class.isAssignableFrom(ob.getValue().getClass())) { - subfieldValue = getFilteredObjectCollection((Collection)ob.getValue(), fields.subList(1, length)); + if (subfields == null) { + valueMap.put(field.getKey(), valueObject); + } else { + if (Collection.class.isAssignableFrom(valueObject.getClass())) { + List subfieldObjectsList = + getFilteredObjectCollection((Collection) valueObject, subfields); + valueMap.put(field.getKey(), subfieldObjectsList); + } else if (Map.class.isAssignableFrom(valueObject.getClass())) { + Map subfieldObjectsMap = new HashMap<>(); + Map valueObjectMap = (Map) valueObject; + for (Map.Entry ob : valueObjectMap.entrySet()) { + Object subfieldValue; + if (Collection.class.isAssignableFrom(ob.getValue().getClass())) { + subfieldValue = getFilteredObjectCollection((Collection)ob.getValue(), subfields); + } else { + subfieldValue = getFilteredObject(ob.getValue(), ob.getValue().getClass(), subfields); + } + subfieldObjectsMap.put(ob.getKey(), subfieldValue); + } + valueMap.put(field.getKey(), subfieldObjectsMap); } else { - subfieldValue = getFilteredObject(ob.getValue(), ob.getValue().getClass(), fields.subList(1, length)); + valueMap.put(field.getKey(), + getFilteredObject(valueObject, valueClassField.getType(), subfields)); } - subfieldObjectsMap.put(ob.getKey(), subfieldValue); } - valueMap.put(fields.get(0), subfieldObjectsMap); - } else { - valueMap.put(fields.get(0), - getFilteredObject(valueObject, valueClassField.getType(), fields.subList(1, length))); + } catch (NoSuchFieldException ex) { + err().println("ERROR: no such field: " + field); + } catch (IllegalAccessException e) { + err().println("ERROR: Cannot get field from object: " + field); } } return valueMap; } - List getFilteredObjectCollection(Collection valueObject, List fields) + List getFilteredObjectCollection(Collection valueObject, Map fields) throws NoSuchFieldException, IllegalAccessException { List subfieldObjectsList = new ArrayList<>(); for (Object ob : valueObject) {