Skip to content

Commit 7a5f2c7

Browse files
committed
Parse and sync OVAL definitions in bulk
1 parent dbaaed3 commit 7a5f2c7

File tree

7 files changed

+169
-71
lines changed

7 files changed

+169
-71
lines changed

java/code/src/com/redhat/rhn/manager/audit/CVEAuditManagerOVAL.java

Lines changed: 22 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -29,11 +29,12 @@
2929

3030
import com.suse.oval.OVALCleaner;
3131
import com.suse.oval.OsFamily;
32-
import com.suse.oval.OvalParser;
3332
import com.suse.oval.config.OVALConfigLoader;
3433
import com.suse.oval.ovaldownloader.OVALDownloadResult;
3534
import com.suse.oval.ovaldownloader.OVALDownloader;
3635
import com.suse.oval.ovaltypes.OvalRootType;
36+
import com.suse.oval.parser.OVALResources;
37+
import com.suse.oval.parser.OvalParser;
3738
import com.suse.oval.vulnerablepkgextractor.VulnerablePackage;
3839

3940
import org.apache.logging.log4j.LogManager;
@@ -345,29 +346,35 @@ public static void syncOVAL() {
345346
LOG.warn("OVAL patch file: " + downloadResult.getPatchFile().map(File::getAbsoluteFile).orElse(null));
346347

347348
downloadResult.getVulnerabilityFile().ifPresent(ovalVulnerabilityFile -> {
348-
OvalParser ovalParser = new OvalParser();
349-
OvalRootType ovalRoot = ovalParser.parse(ovalVulnerabilityFile);
350-
351-
LOG.warn("Saving Vulnerability OVAL for {} {}", product.getOsFamily(), product.getOsVersion());
352-
353-
OVALCleaner.cleanup(ovalRoot, product.getOsFamily(), product.getOsVersion());
354-
OVALCachingFactory.savePlatformsVulnerablePackages(ovalRoot);
349+
LOG.warn("Syncing Vulnerability OVAL for {} {}",
350+
product.getOsFamily(), product.getOsVersion());
351+
syncOVALFromFile(product, ovalVulnerabilityFile);
355352
});
356353

357354
downloadResult.getPatchFile().ifPresent(patchFile -> {
358-
OvalParser ovalParser = new OvalParser();
359-
OvalRootType ovalRoot = ovalParser.parse(patchFile);
360-
361-
LOG.warn("Saving Patch OVAL for {} {}", product.getOsFamily(), product.getOsVersion());
362-
363-
OVALCleaner.cleanup(ovalRoot, product.getOsFamily(), product.getOsVersion());
364-
OVALCachingFactory.savePlatformsVulnerablePackages(ovalRoot);
355+
LOG.warn("Syncing Patch OVAL for {} {}", product.getOsFamily(), product.getOsVersion());
356+
syncOVALFromFile(product, patchFile);
365357
});
366358

367359
LOG.warn("Saving OVAL finished");
368360
}
369361
}
370362

363+
private static void syncOVALFromFile(OVALProduct product, File ovalVulnerabilityFile) {
364+
OvalParser ovalParser = new OvalParser();
365+
OVALResources ovalResources = ovalParser.parseResources(ovalVulnerabilityFile);
366+
ovalParser.parseDefinitionsInBulk(ovalVulnerabilityFile, (definitionsBulk) -> {
367+
OvalRootType ovalRoot = new OvalRootType();
368+
ovalRoot.setDefinitions(definitionsBulk);
369+
ovalRoot.setTests(ovalResources.getTests());
370+
ovalRoot.setObjects(ovalResources.getObjects());
371+
ovalRoot.setStates(ovalResources.getStates());
372+
373+
OVALCleaner.cleanup(ovalRoot, product.getOsFamily(), product.getOsVersion());
374+
OVALCachingFactory.savePlatformsVulnerablePackages(ovalRoot);
375+
});
376+
}
377+
371378
/**
372379
* Identifies the OS products to synchronize OVAL data for.
373380
* */

java/code/src/com/redhat/rhn/manager/audit/test/CVEAuditManagerOVALTest.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@
4747
import com.suse.oval.OVALCachingFactory;
4848
import com.suse.oval.OVALCleaner;
4949
import com.suse.oval.OsFamily;
50-
import com.suse.oval.OvalParser;
50+
import com.suse.oval.parser.OvalParser;
5151
import com.suse.oval.ovaltypes.OvalRootType;
5252

5353
import org.apache.logging.log4j.LogManager;

java/code/src/com/suse/oval/exceptions/OvalParserException.java

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,10 @@
1515

1616
package com.suse.oval.exceptions;
1717

18+
import com.suse.oval.parser.OvalParser;
19+
1820
/**
19-
* A runtime exception throne by {@link com.suse.oval.OvalParser} when it encounters errors while parsing an OVAL file
21+
* A runtime exception throne by {@link OvalParser} when it encounters errors while parsing an OVAL file
2022
* */
2123
public class OvalParserException extends RuntimeException {
2224

java/code/src/com/suse/oval/manager/OVALResourcesCache.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515

1616
package com.suse.oval.manager;
1717

18-
import com.suse.oval.parser.OVALResourcesResult;
18+
import com.suse.oval.parser.OVALResources;
1919
import com.suse.oval.ovaltypes.ObjectType;
2020
import com.suse.oval.ovaltypes.OvalRootType;
2121
import com.suse.oval.ovaltypes.StateType;
@@ -39,7 +39,7 @@ public OVALResourcesCache(OvalRootType rootType) {
3939
this.objectManager = new OvalObjectManager(rootType.getObjects());
4040
}
4141

42-
public OVALResourcesCache(OVALResourcesResult resourcesResult) {
42+
public OVALResourcesCache(OVALResources resourcesResult) {
4343
this.stateManager = new OvalStateManager(resourcesResult.getStates());
4444
this.testManager = new OvalTestManager(resourcesResult.getTests());
4545
this.objectManager = new OvalObjectManager(resourcesResult.getObjects());
Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
/*
2+
* Copyright (c) 2023 SUSE LLC
3+
*
4+
* This software is licensed to you under the GNU General Public License,
5+
* version 2 (GPLv2). There is NO WARRANTY for this software, express or
6+
* implied, including the implied warranties of MERCHANTABILITY or FITNESS
7+
* FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2
8+
* along with this software; if not, see
9+
* http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt.
10+
*
11+
* Red Hat trademarks are not licensed under GPLv2. No permission is
12+
* granted to use or replicate Red Hat trademarks that are incorporated
13+
* in this software or its documentation.
14+
*/
15+
16+
package com.suse.oval.parser;
17+
18+
import java.util.List;
19+
20+
import com.suse.oval.manager.OVALResourcesCache;
21+
import com.suse.oval.ovaltypes.DefinitionType;
22+
23+
@FunctionalInterface
24+
public interface OVALDefinitionsBulkHandler {
25+
void handle(List<DefinitionType> bulk);
26+
}
Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
/*
2+
* Copyright (c) 2023 SUSE LLC
3+
*
4+
* This software is licensed to you under the GNU General Public License,
5+
* version 2 (GPLv2). There is NO WARRANTY for this software, express or
6+
* implied, including the implied warranties of MERCHANTABILITY or FITNESS
7+
* FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2
8+
* along with this software; if not, see
9+
* http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt.
10+
*
11+
* Red Hat trademarks are not licensed under GPLv2. No permission is
12+
* granted to use or replicate Red Hat trademarks that are incorporated
13+
* in this software or its documentation.
14+
*/
15+
16+
package com.suse.oval.parser;
17+
18+
import java.util.ArrayList;
19+
import java.util.List;
20+
21+
import com.suse.oval.manager.OVALResourcesCache;
22+
import com.suse.oval.ovaltypes.ObjectType;
23+
import com.suse.oval.ovaltypes.StateType;
24+
import com.suse.oval.ovaltypes.TestType;
25+
26+
public class OVALResources {
27+
private List<ObjectType> objects = new ArrayList<>();
28+
private List<StateType> states = new ArrayList<>();
29+
private List<TestType> tests = new ArrayList<>();
30+
31+
public List<ObjectType> getObjects() {
32+
return objects;
33+
}
34+
35+
public List<StateType> getStates() {
36+
return states;
37+
}
38+
39+
public List<TestType> getTests() {
40+
return tests;
41+
}
42+
43+
public void setObjects(List<ObjectType> objects) {
44+
this.objects = objects;
45+
}
46+
47+
public void setStates(List<StateType> states) {
48+
this.states = states;
49+
}
50+
51+
public void setTests(List<TestType> tests) {
52+
this.tests = tests;
53+
}
54+
55+
public OVALResourcesCache getCache() {
56+
return new OVALResourcesCache(this);
57+
}
58+
}

java/code/src/com/suse/oval/OvalParser.java renamed to java/code/src/com/suse/oval/parser/OvalParser.java

Lines changed: 57 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@
1313
* in this software or its documentation.
1414
*/
1515

16-
package com.suse.oval;
16+
package com.suse.oval.parser;
1717

1818
import com.redhat.rhn.domain.rhnpackage.PackageType;
1919
import com.suse.oval.exceptions.OvalParserException;
@@ -60,16 +60,17 @@
6060
import java.io.File;
6161
import java.io.FileInputStream;
6262
import java.io.FileNotFoundException;
63-
import java.net.URISyntaxException;
6463
import java.net.URL;
65-
import java.util.List;
64+
6665
import java.util.ArrayList;
66+
import java.util.List;
6767
import java.util.Objects;
6868

6969
/**
7070
* The Oval Parser is responsible for parsing OVAL(Open Vulnerability and Assessment Language) documents
7171
*/
7272
public class OvalParser {
73+
private static final int DEFINITIONS_BULK_SIZE = 500;
7374
private static final Logger LOG = LogManager.getLogger(OvalParser.class);
7475
private static final List<String> TEST_TYPES = List.of("rpminfo_test", "dpkginfo_test");
7576
private static final List<String> OBJECT_TYPES = List.of("rpminfo_object", "dpkginfo_object");
@@ -81,38 +82,70 @@ public class OvalParser {
8182
* @param ovalFile the OVAL file to parse
8283
* @return the parsed OVAL encapulated in a {@link OvalRootType} object=
8384
* */
84-
public OvalRootType parse(File ovalFile) throws OvalParserException {
85+
public OvalRootType parse(URL ovalFile) throws OvalParserException {
8586
try {
8687
JAXBContext jaxbContext = JAXBContext.newInstance(OvalRootType.class);
8788
Unmarshaller unmarshaller = jaxbContext.createUnmarshaller();
8889
return (OvalRootType) unmarshaller.unmarshal(ovalFile);
8990
}
9091
catch (JAXBException e) {
91-
throw new OvalParserException("Failed to parse the given OVAL file at: " + ovalFile.getAbsolutePath(), e);
92+
throw new OvalParserException("Failed to parse the given OVAL file at: " + ovalFile, e);
9293
}
9394
}
9495

95-
/**
96-
* Parse the given OVAL file from a URL
97-
*
98-
* @param url the URL to get the OVAL file from
99-
* @return the parsed OVAL encapsulated in a {@link OvalRootType} object
100-
* */
101-
public OvalRootType parse(URL url) {
96+
public void parseDefinitionsInBulk(File ovalFile, OVALDefinitionsBulkHandler bulkHandler) {
97+
XMLInputFactory xmlInputFactory = XMLInputFactory.newInstance();
10298
try {
103-
return parseStax(new File(url.toURI()));
99+
XMLEventReader reader = xmlInputFactory.createXMLEventReader(new FileInputStream(ovalFile));
100+
101+
List<DefinitionType> definitions = new ArrayList<>();
102+
103+
while (reader.hasNext()) {
104+
XMLEvent nextEvent = reader.nextEvent();
105+
106+
if (nextEvent.isStartElement()) {
107+
String elementName = nextEvent.asStartElement().getName().getLocalPart();
108+
if (elementName.equals("definition")) {
109+
DefinitionType definitionType = parseDefinitionType(nextEvent.asStartElement(), reader);
110+
definitions.add(definitionType);
111+
112+
if (definitions.size() == DEFINITIONS_BULK_SIZE) {
113+
bulkHandler.handle(definitions);
114+
definitions = new ArrayList<>();
115+
}
116+
}
117+
}
118+
119+
if (nextEvent.isEndElement()) {
120+
if (nextEvent.asEndElement().getName().getLocalPart().equals("definitions")) {
121+
if (!definitions.isEmpty()) {
122+
bulkHandler.handle(definitions);
123+
definitions = new ArrayList<>();
124+
}
125+
}
126+
}
127+
}
104128
}
105-
catch (URISyntaxException e) {
106-
throw new RuntimeException(e);
129+
catch (XMLStreamException | FileNotFoundException e) {
130+
throw new OvalParserException("Failed to parse OVAL definitions from OVAL file at: "
131+
+ ovalFile.getAbsolutePath(), e);
107132
}
108133
}
109134

110-
public OvalRootType parseStax(File ovalFile) {
135+
public List<DefinitionType> parseAllDefinitions(File ovalFile) {
136+
List<DefinitionType> allDefinitions = new ArrayList<>();
137+
138+
parseDefinitionsInBulk(ovalFile, allDefinitions::addAll);
139+
140+
return allDefinitions;
141+
}
142+
143+
public OVALResources parseResources(File ovalFile) {
111144
XMLInputFactory xmlInputFactory = XMLInputFactory.newInstance();
112145
try {
113146
XMLEventReader reader = xmlInputFactory.createXMLEventReader(new FileInputStream(ovalFile));
114147

115-
OvalRootType ovalRoot = new OvalRootType();
148+
OVALResources resources = new OVALResources();
116149

117150
while (reader.hasNext()) {
118151
XMLEvent nextEvent = reader.nextEvent();
@@ -121,54 +154,28 @@ public OvalRootType parseStax(File ovalFile) {
121154
String elementName = nextEvent.asStartElement().getName().getLocalPart();
122155
switch (elementName) {
123156
case "objects":
124-
ovalRoot.setObjects(parseObjects(reader));
157+
resources.setObjects(parseObjects(reader));
125158
break;
126159
case "states":
127-
ovalRoot.setStates(parseStates(reader));
160+
resources.setStates(parseStates(reader));
128161
break;
129162
case "tests":
130-
ovalRoot.setTests(parseTests(reader));
131-
break;
132-
case "definitions":
133-
ovalRoot.setDefinitions(parseDefinitions(reader));
163+
resources.setTests(parseTests(reader));
134164
break;
135165
default: // Do nothing
136166
}
137167
}
138168
}
139169

140-
return ovalRoot;
141-
170+
return resources;
142171
}
143172
catch (XMLStreamException | FileNotFoundException e) {
144-
throw new OvalParserException("Failed to parse the given OVAL file at: " + ovalFile.getAbsolutePath(), e);
173+
throw new OvalParserException(
174+
"Failed to parse the OVAL resources(tests, states and objects) from OVAL file at: " +
175+
ovalFile.getAbsolutePath(), e);
145176
}
146177
}
147178

148-
private List<DefinitionType> parseDefinitions(XMLEventReader reader) throws XMLStreamException {
149-
List<DefinitionType> definitions = new ArrayList<>();
150-
151-
while (reader.hasNext()) {
152-
XMLEvent nextEvent = reader.nextEvent();
153-
154-
if (nextEvent.isStartElement()) {
155-
if (nextEvent.asStartElement().getName().getLocalPart().equals("definition")) {
156-
DefinitionType definitionType = parseDefinitionType(nextEvent.asStartElement(), reader);
157-
definitions.add(definitionType);
158-
}
159-
}
160-
161-
if (nextEvent.isEndElement()) {
162-
if (nextEvent.asEndElement().getName().getLocalPart().equals("definitions")) {
163-
return definitions;
164-
}
165-
}
166-
}
167-
168-
throw new OvalParserException("Unable to find the closing tag for </definitions>");
169-
170-
}
171-
172179
private DefinitionType parseDefinitionType(StartElement definitionElement, XMLEventReader reader)
173180
throws XMLStreamException {
174181
DefinitionType definitionType = new DefinitionType();
@@ -541,7 +548,6 @@ private StateType parseStateType(StartElement rpmStateElement, XMLEventReader re
541548

542549
rpmStateElement.getAttributes().forEachRemaining(attribute -> {
543550
String attributeName = attribute.getName().getLocalPart();
544-
545551
switch (attributeName) {
546552
case "id":
547553
stateType.setId(attribute.getValue());
@@ -679,7 +685,6 @@ private ObjectType parseObjectType(StartElement rpmObjectElement, XMLEventReader
679685
}
680686
});
681687

682-
683688
while (reader.hasNext()) {
684689
XMLEvent nextEvent = reader.nextEvent();
685690
if (nextEvent.isStartElement()) {

0 commit comments

Comments
 (0)