From 4c83051dfbe67509fc22671a511aa0e595a78687 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominique=20J=C3=A4ggi?= Date: Wed, 29 Nov 2023 09:41:53 +0100 Subject: [PATCH 01/28] feat: add spacecat-shared-data-access package --- package-lock.json | 36 ++- .../spacecat-shared-data-access/.jsdoc.json | 17 ++ .../.mocha-multi.json | 6 + .../spacecat-shared-data-access/.npmignore | 9 + .../spacecat-shared-data-access/.nycrc.json | 10 + .../spacecat-shared-data-access/CHANGELOG.md | 0 .../spacecat-shared-data-access/LICENSE.txt | 264 ++++++++++++++++++ .../spacecat-shared-data-access/README.md | 2 + .../spacecat-shared-data-access/package.json | 37 +++ packages/spacecat-shared-dynamo/package.json | 2 +- 10 files changed, 367 insertions(+), 16 deletions(-) create mode 100644 packages/spacecat-shared-data-access/.jsdoc.json create mode 100644 packages/spacecat-shared-data-access/.mocha-multi.json create mode 100644 packages/spacecat-shared-data-access/.npmignore create mode 100644 packages/spacecat-shared-data-access/.nycrc.json create mode 100644 packages/spacecat-shared-data-access/CHANGELOG.md create mode 100644 packages/spacecat-shared-data-access/LICENSE.txt create mode 100644 packages/spacecat-shared-data-access/README.md create mode 100644 packages/spacecat-shared-data-access/package.json diff --git a/package-lock.json b/package-lock.json index 839be8fe9..b37bce2d0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -29,7 +29,7 @@ "nock": "13.3.8", "semantic-release": "19.0.5", "semantic-release-monorepo": "7.0.5", - "typescript": "^5.3.2" + "typescript": "5.3.2" } }, "node_modules/@aashutoshrathi/word-wrap": { @@ -89,12 +89,15 @@ "version": "4.4.1", "resolved": "https://registry.npmjs.org/@adobe/helix-universal/-/helix-universal-4.4.1.tgz", "integrity": "sha512-YJKiuzeAZvzkardlAVcyW/PYNzKfIf/3v7Dd22wxvimmI3gct4k3V39gUSVrP+sWZPvlBS67WMNCv28n0eKjWg==", - "optional": true, "dependencies": { "@adobe/fetch": "4.1.1", "aws4": "1.12.0" } }, + "node_modules/@adobe/spacecat-shared-data-access": { + "resolved": "packages/spacecat-shared-data-access", + "link": true + }, "node_modules/@adobe/spacecat-shared-dynamo": { "resolved": "packages/spacecat-shared-dynamo", "link": true @@ -12097,12 +12100,22 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "packages/spacecat-shared-data-access": { + "version": "1.1.0", + "license": "Apache-2.0", + "dependencies": { + "@adobe/spacecat-shared-utils": "1.1.0" + }, + "devDependencies": { + "chai": "4.3.10" + } + }, "packages/spacecat-shared-dynamo": { "name": "@adobe/spacecat-shared-dynamo", - "version": "1.0.0", + "version": "1.1.0", "license": "Apache-2.0", "dependencies": { - "@adobe/spacecat-shared-utils": "1.0.1", + "@adobe/spacecat-shared-utils": "1.1.0", "@aws-sdk/client-dynamodb": "3.454.0", "@aws-sdk/lib-dynamodb": "3.454.0" }, @@ -12110,28 +12123,21 @@ "chai": "4.3.10" } }, - "packages/spacecat-shared-dynamo/node_modules/@adobe/spacecat-shared-utils": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/@adobe/spacecat-shared-utils/-/spacecat-shared-utils-1.0.1.tgz", - "integrity": "sha512-6HiCywan5y9jpbFYkQX87Vg0q5dXoJf2m6MiDvNBEJdREdUxFwE+oYA6Fr86qqd32ECnCeYCFO13HcNRn1RxkQ==" - }, "packages/spacecat-shared-example": { "name": "@adobe/spacecat-shared-example", - "version": "1.0.0", + "version": "1.1.0", "license": "Apache-2.0", "dependencies": { "@adobe/fetch": "4.1.1", "@adobe/helix-shared-wrap": "2.0.0", + "@adobe/helix-universal": "4.4.1", "aws4": "1.12.0" }, - "devDependencies": {}, - "optionalDependencies": { - "@adobe/helix-universal": "4.4.1" - } + "devDependencies": {} }, "packages/spacecat-shared-utils": { "name": "@adobe/spacecat-shared-utils", - "version": "1.0.1", + "version": "1.1.0", "license": "Apache-2.0", "devDependencies": { "chai": "4.3.10" diff --git a/packages/spacecat-shared-data-access/.jsdoc.json b/packages/spacecat-shared-data-access/.jsdoc.json new file mode 100644 index 000000000..405090f4b --- /dev/null +++ b/packages/spacecat-shared-data-access/.jsdoc.json @@ -0,0 +1,17 @@ +{ + "plugins": [], + "recurseDepth": 10, + "source": { + "includePattern": ".+\\.js(doc|x)?$", + "excludePattern": "(^|\\/|\\\\)_" + }, + "sourceType": "module", + "tags": { + "allowUnknownTags": true, + "dictionaries": ["jsdoc","closure"] + }, + "templates": { + "cleverLinks": false, + "monospaceLinks": false + } +} \ No newline at end of file diff --git a/packages/spacecat-shared-data-access/.mocha-multi.json b/packages/spacecat-shared-data-access/.mocha-multi.json new file mode 100644 index 000000000..aa2be2a23 --- /dev/null +++ b/packages/spacecat-shared-data-access/.mocha-multi.json @@ -0,0 +1,6 @@ +{ + "reporterEnabled": "spec,xunit", + "xunitReporterOptions": { + "output": "junit/test-results.xml" + } +} diff --git a/packages/spacecat-shared-data-access/.npmignore b/packages/spacecat-shared-data-access/.npmignore new file mode 100644 index 000000000..868317d21 --- /dev/null +++ b/packages/spacecat-shared-data-access/.npmignore @@ -0,0 +1,9 @@ +coverage/ +node_modules/ +junit/ +test/ +docs/ +logs/ +test-results.xml +renovate.json +.* diff --git a/packages/spacecat-shared-data-access/.nycrc.json b/packages/spacecat-shared-data-access/.nycrc.json new file mode 100644 index 000000000..78e7a0b14 --- /dev/null +++ b/packages/spacecat-shared-data-access/.nycrc.json @@ -0,0 +1,10 @@ +{ + "reporter": [ + "lcov", + "text" + ], + "check-coverage": true, + "lines": 100, + "branches": 97, + "statements": 100 +} diff --git a/packages/spacecat-shared-data-access/CHANGELOG.md b/packages/spacecat-shared-data-access/CHANGELOG.md new file mode 100644 index 000000000..e69de29bb diff --git a/packages/spacecat-shared-data-access/LICENSE.txt b/packages/spacecat-shared-data-access/LICENSE.txt new file mode 100644 index 000000000..883ab098f --- /dev/null +++ b/packages/spacecat-shared-data-access/LICENSE.txt @@ -0,0 +1,264 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + +APACHE JACKRABBIT SUBCOMPONENTS + +Apache Jackrabbit includes parts with separate copyright notices and license +terms. Your use of these subcomponents is subject to the terms and conditions +of the following licenses: + + XPath 2.0/XQuery 1.0 Parser: + http://www.w3.org/2002/11/xquery-xpath-applets/xgrammar.zip + + Copyright (C) 2002 World Wide Web Consortium, (Massachusetts Institute of + Technology, European Research Consortium for Informatics and Mathematics, + Keio University). All Rights Reserved. + + This work is distributed under the W3C(R) Software License in the hope + that it will be useful, but WITHOUT ANY WARRANTY; without even the + implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + + W3C(R) SOFTWARE NOTICE AND LICENSE + http://www.w3.org/Consortium/Legal/2002/copyright-software-20021231 + + This work (and included software, documentation such as READMEs, or + other related items) is being provided by the copyright holders under + the following license. By obtaining, using and/or copying this work, + you (the licensee) agree that you have read, understood, and will comply + with the following terms and conditions. + + Permission to copy, modify, and distribute this software and its + documentation, with or without modification, for any purpose and + without fee or royalty is hereby granted, provided that you include + the following on ALL copies of the software and documentation or + portions thereof, including modifications: + + 1. The full text of this NOTICE in a location viewable to users + of the redistributed or derivative work. + + 2. Any pre-existing intellectual property disclaimers, notices, + or terms and conditions. If none exist, the W3C Software Short + Notice should be included (hypertext is preferred, text is + permitted) within the body of any redistributed or derivative code. + + 3. Notice of any changes or modifications to the files, including + the date changes were made. (We recommend you provide URIs to the + location from which the code is derived.) + + THIS SOFTWARE AND DOCUMENTATION IS PROVIDED "AS IS," AND COPYRIGHT + HOLDERS MAKE NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR IMPLIED, + INCLUDING BUT NOT LIMITED TO, WARRANTIES OF MERCHANTABILITY OR FITNESS + FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF THE SOFTWARE OR + DOCUMENTATION WILL NOT INFRINGE ANY THIRD PARTY PATENTS, COPYRIGHTS, + TRADEMARKS OR OTHER RIGHTS. + + COPYRIGHT HOLDERS WILL NOT BE LIABLE FOR ANY DIRECT, INDIRECT, SPECIAL + OR CONSEQUENTIAL DAMAGES ARISING OUT OF ANY USE OF THE SOFTWARE OR + DOCUMENTATION. + + The name and trademarks of copyright holders may NOT be used in + advertising or publicity pertaining to the software without specific, + written prior permission. Title to copyright in this software and + any associated documentation will at all times remain with + copyright holders. diff --git a/packages/spacecat-shared-data-access/README.md b/packages/spacecat-shared-data-access/README.md new file mode 100644 index 000000000..e23e5be9e --- /dev/null +++ b/packages/spacecat-shared-data-access/README.md @@ -0,0 +1,2 @@ +# Data Access Module + diff --git a/packages/spacecat-shared-data-access/package.json b/packages/spacecat-shared-data-access/package.json new file mode 100644 index 000000000..028e9cff9 --- /dev/null +++ b/packages/spacecat-shared-data-access/package.json @@ -0,0 +1,37 @@ +{ + "name": "@adobe/spacecat-shared-data-access", + "version": "1.1.0", + "description": "Shared modules of the Spacecat Services - Data Access", + "type": "module", + "main": "src/index.js", + "types": "src/index.d.ts", + "scripts": { + "test": "c8 mocha", + "lint": "eslint .", + "clean": "rm -rf package-lock.json node_modules" + }, + "mocha": { + "reporter": "mocha-multi-reporters", + "reporter-options": "configFile=.mocha-multi.json", + "spec": "test/**/*.test.js" + }, + "repository": { + "type": "git", + "url": "https://github.com/adobe-rnd/spacecat-shared.git" + }, + "author": "", + "license": "Apache-2.0", + "bugs": { + "url": "https://github.com/adobe-rnd/spacecat-shared/issues" + }, + "homepage": "https://github.com/adobe-rnd/spacecat-shared#readme", + "publishConfig": { + "access": "public" + }, + "dependencies": { + "@adobe/spacecat-shared-utils": "1.1.0" + }, + "devDependencies": { + "chai": "4.3.10" + } +} diff --git a/packages/spacecat-shared-dynamo/package.json b/packages/spacecat-shared-dynamo/package.json index bef51cb40..5cb4bc13b 100644 --- a/packages/spacecat-shared-dynamo/package.json +++ b/packages/spacecat-shared-dynamo/package.json @@ -31,7 +31,7 @@ "dependencies": { "@aws-sdk/client-dynamodb": "3.454.0", "@aws-sdk/lib-dynamodb": "3.454.0", - "@adobe/spacecat-shared-utils": "1.0.1" + "@adobe/spacecat-shared-utils": "1.1.0" }, "devDependencies": { "chai": "4.3.10" From 9e93178e8ad0d3ff4bafd672486502a687f26f96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominique=20J=C3=A4ggi?= Date: Wed, 29 Nov 2023 10:45:19 +0100 Subject: [PATCH 02/28] fix: key validation --- .../src/utils/guards.js | 8 +++--- .../test/modules/getItem.test.js | 2 +- .../test/modules/removeItem.test.js | 2 +- .../test/utils/guards.test.js | 28 ++++++++++--------- 4 files changed, 21 insertions(+), 19 deletions(-) diff --git a/packages/spacecat-shared-dynamo/src/utils/guards.js b/packages/spacecat-shared-dynamo/src/utils/guards.js index 62b392721..11d52bcc3 100644 --- a/packages/spacecat-shared-dynamo/src/utils/guards.js +++ b/packages/spacecat-shared-dynamo/src/utils/guards.js @@ -25,14 +25,14 @@ const guardTableName = (tableName) => { }; /** - * Validates that the provided key is an object and contains a partitionKey. + * Validates that the provided key is an object and contains at least one property. * * @param {object} key - The key object to validate. - * @throws {Error} If the key is not an object or does not contain a partitionKey. + * @throws {Error} If the key is not an object or does not contain at least one property. */ const guardKey = (key) => { - if (!isObject(key) || !key.partitionKey) { - throw new Error('Key must be an object with a partitionKey.'); + if (!isObject(key) || Object.keys(key).length === 0) { + throw new Error('Key must be a non-empty object.'); } }; diff --git a/packages/spacecat-shared-dynamo/test/modules/getItem.test.js b/packages/spacecat-shared-dynamo/test/modules/getItem.test.js index f89a131b3..205c0101b 100644 --- a/packages/spacecat-shared-dynamo/test/modules/getItem.test.js +++ b/packages/spacecat-shared-dynamo/test/modules/getItem.test.js @@ -54,7 +54,7 @@ describe('getItem', () => { await dynamoDbClient.getItem('TestTable', null); expect.fail('getItem did not throw with invalid key'); } catch (error) { - expect(error.message).to.equal('Key must be an object with a partitionKey.'); + expect(error.message).to.equal('Key must be a non-empty object.'); } }); diff --git a/packages/spacecat-shared-dynamo/test/modules/removeItem.test.js b/packages/spacecat-shared-dynamo/test/modules/removeItem.test.js index 37f94fd63..02e551661 100644 --- a/packages/spacecat-shared-dynamo/test/modules/removeItem.test.js +++ b/packages/spacecat-shared-dynamo/test/modules/removeItem.test.js @@ -54,7 +54,7 @@ describe('removeItem', () => { await dynamoDbClient.removeItem('TestTable', null); expect.fail('removeItem did not throw with invalid key'); } catch (error) { - expect(error.message).to.equal('Key must be an object with a partitionKey.'); + expect(error.message).to.equal('Key must be a non-empty object.'); } }); diff --git a/packages/spacecat-shared-dynamo/test/utils/guards.test.js b/packages/spacecat-shared-dynamo/test/utils/guards.test.js index 814b2c4db..26846b338 100644 --- a/packages/spacecat-shared-dynamo/test/utils/guards.test.js +++ b/packages/spacecat-shared-dynamo/test/utils/guards.test.js @@ -16,42 +16,44 @@ import { expect } from 'chai'; import { guardKey, guardQueryParameters, guardTableName } from '../../src/utils/guards.js'; describe('Query Parameter Guards', () => { - // Test guardTableName describe('guardTableName', () => { - it('should throw an error if tableName is empty', () => { + it('throws an error if tableName is empty', () => { expect(() => guardTableName('')).to.throw('Table name is required.'); }); - it('should not throw an error for valid tableName', () => { + it('does not throw an error for valid tableName', () => { expect(() => guardTableName('validTableName')).to.not.throw(); }); }); - // Test guardKey describe('guardKey', () => { - it('should throw an error if key is not an object', () => { - expect(() => guardKey('notAnObject')).to.throw('Key must be an object with a partitionKey.'); + it('throws an error if key is not an object', () => { + expect(() => guardKey('notAnObject')).to.throw('Key must be a non-empty object.'); }); - it('should throw an error if key does not have partitionKey', () => { - expect(() => guardKey({})).to.throw('Key must be an object with a partitionKey.'); + it('throws an error if key is an empty object', () => { + expect(() => guardKey({})).to.throw('Key must be a non-empty object.'); }); - it('should not throw an error for a valid key', () => { - expect(() => guardKey({ partitionKey: 'value' })).to.not.throw(); + it('does not throw an error for a valid key with one property', () => { + expect(() => guardKey({ somePartitionKeyField: 'value' })).to.not.throw(); + }); + + it('does not throw an error for a valid key with two properties', () => { + expect(() => guardKey({ somePartitionKeyField: 'value', someOptionalRangeKey: 'anotherValue' })).to.not.throw(); }); }); describe('guardQueryParameters', () => { - it('should throw an error if params is not an object', () => { + it('throws an error if params is not an object', () => { expect(() => guardQueryParameters('notAnObject')).to.throw('Query parameters must be an object.'); }); - it('should throw an error if any required parameter is missing', () => { + it('throws an error if any required parameter is missing', () => { expect(() => guardQueryParameters({ TableName: 'table' })).to.throw('Query parameters is missing required parameter: KeyConditionExpression'); }); - it('should not throw an error for valid params', () => { + it('does not throw an error for valid params', () => { const validParams = { TableName: 'table', KeyConditionExpression: 'expression', From 49263dc14f30016e6eb052df00e71bc593ee855c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominique=20J=C3=A4ggi?= Date: Wed, 29 Nov 2023 18:44:35 +0100 Subject: [PATCH 03/28] feat: add models, access patterns and tests --- package-lock.json | 175 ++++++++++- .../docs/schema.json | 290 ++++++++++++++++++ .../spacecat-shared-data-access/package.json | 7 +- .../src/audits/accessPatterns.js | 95 ++++++ .../src/audits/index.js | 34 ++ .../src/index.d.ts | 71 +++++ .../spacecat-shared-data-access/src/index.js | 33 ++ .../src/models/audit.js | 60 ++++ .../src/models/base.js | 30 ++ .../src/models/site.js | 51 +++ .../src/sites/accessPatterns.js | 170 ++++++++++ .../src/sites/index.js | 59 ++++ .../test/audits/index.test.js | 79 +++++ .../test/index.test.js | 58 ++++ .../test/models/audit.test.js | 69 +++++ .../test/models/base.test.js | 57 ++++ .../test/models/site.test.js | 65 ++++ .../test/sites/index.test.js | 214 +++++++++++++ 18 files changed, 1612 insertions(+), 5 deletions(-) create mode 100644 packages/spacecat-shared-data-access/docs/schema.json create mode 100644 packages/spacecat-shared-data-access/src/audits/accessPatterns.js create mode 100644 packages/spacecat-shared-data-access/src/audits/index.js create mode 100644 packages/spacecat-shared-data-access/src/index.d.ts create mode 100644 packages/spacecat-shared-data-access/src/index.js create mode 100644 packages/spacecat-shared-data-access/src/models/audit.js create mode 100644 packages/spacecat-shared-data-access/src/models/base.js create mode 100644 packages/spacecat-shared-data-access/src/models/site.js create mode 100644 packages/spacecat-shared-data-access/src/sites/accessPatterns.js create mode 100644 packages/spacecat-shared-data-access/src/sites/index.js create mode 100644 packages/spacecat-shared-data-access/test/audits/index.test.js create mode 100644 packages/spacecat-shared-data-access/test/index.test.js create mode 100644 packages/spacecat-shared-data-access/test/models/audit.test.js create mode 100644 packages/spacecat-shared-data-access/test/models/base.test.js create mode 100644 packages/spacecat-shared-data-access/test/models/site.test.js create mode 100644 packages/spacecat-shared-data-access/test/sites/index.test.js diff --git a/package-lock.json b/package-lock.json index b37bce2d0..0e9459ca5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1491,6 +1491,50 @@ "semantic-release": ">=18.0.0-beta.1" } }, + "node_modules/@sinonjs/commons": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-3.0.0.tgz", + "integrity": "sha512-jXBtWAF4vmdNmZgD5FoKsVLv3rPgDnLgPbU84LIJ3otV44vJlDRokVng5v8NFJdCf/da9legHcKaRuZs4L7faA==", + "dev": true, + "dependencies": { + "type-detect": "4.0.8" + } + }, + "node_modules/@sinonjs/fake-timers": { + "version": "11.2.2", + "resolved": "https://registry.npmjs.org/@sinonjs/fake-timers/-/fake-timers-11.2.2.tgz", + "integrity": "sha512-G2piCSxQ7oWOxwGSAyFHfPIsyeJGXYtc6mFbnFA+kRXkiEnTl8c/8jul2S329iFBnDI9HGoeWWAZvuvOkZccgw==", + "dev": true, + "dependencies": { + "@sinonjs/commons": "^3.0.0" + } + }, + "node_modules/@sinonjs/samsam": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/@sinonjs/samsam/-/samsam-8.0.0.tgz", + "integrity": "sha512-Bp8KUVlLp8ibJZrnvq2foVhP0IVX2CIprMJPK0vqGqgrDa0OHVKeZyBykqskkrdxV6yKBPmGasO8LVjAKR3Gew==", + "dev": true, + "dependencies": { + "@sinonjs/commons": "^2.0.0", + "lodash.get": "^4.4.2", + "type-detect": "^4.0.8" + } + }, + "node_modules/@sinonjs/samsam/node_modules/@sinonjs/commons": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-2.0.0.tgz", + "integrity": "sha512-uLa0j859mMrg2slwQYdO/AkrOfmH+X6LTVmNTS9CqexuE2IvVORIkSpJLqePAbEnKJ77aMmCwr1NUZ57120Xcg==", + "dev": true, + "dependencies": { + "type-detect": "4.0.8" + } + }, + "node_modules/@sinonjs/text-encoding": { + "version": "0.7.2", + "resolved": "https://registry.npmjs.org/@sinonjs/text-encoding/-/text-encoding-0.7.2.tgz", + "integrity": "sha512-sXXKG+uL9IrKqViTtao2Ws6dy0znu9sOaP1di/jKGW1M6VssO8vlpXCQcpZ+jisQ1tTFAC5Jo/EOzFbggBagFQ==", + "dev": true + }, "node_modules/@smithy/abort-controller": { "version": "2.0.14", "resolved": "https://registry.npmjs.org/@smithy/abort-controller/-/abort-controller-2.0.14.tgz", @@ -5588,6 +5632,12 @@ "node": "*" } }, + "node_modules/just-extend": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/just-extend/-/just-extend-4.2.1.tgz", + "integrity": "sha512-g3UB796vUFIY90VIv/WX3L2c8CS2MdWUww3CNrYmqza1Fg0DURc2K/O4YrnklBdQarSJ/y8JnJYDGc+1iumQjg==", + "dev": true + }, "node_modules/keyv": { "version": "4.5.4", "resolved": "https://registry.npmjs.org/keyv/-/keyv-4.5.4.tgz", @@ -5909,6 +5959,12 @@ "integrity": "sha512-TM9YBvyC84ZxE3rgfefxUWiQKLilstD6k7PTGt6wfbtXF8ixIJLOL3VYyV/z+ZiPLsVxAsKAFVwWlWeb2Y8Yyw==", "dev": true }, + "node_modules/lodash.get": { + "version": "4.4.2", + "resolved": "https://registry.npmjs.org/lodash.get/-/lodash.get-4.4.2.tgz", + "integrity": "sha512-z+Uw/vLuy6gQe8cfaFWD7p0wVv8fJl3mbzXh33RS+0oW2wvUqiRXiQ69gLWSLpgB5/6sU+r6BlQR0MBILadqTQ==", + "dev": true + }, "node_modules/lodash.ismatch": { "version": "4.4.0", "resolved": "https://registry.npmjs.org/lodash.ismatch/-/lodash.ismatch-4.4.0.tgz", @@ -6620,6 +6676,46 @@ "integrity": "sha512-EZSPZB70jiVsivaBLYDCyntd5eH8NTSMOn3rB+HxwdmKThGELLdYv8qVIMWvZEFy9w8ZZpW9h9OB32l1rGtj7g==", "dev": true }, + "node_modules/nise": { + "version": "5.1.5", + "resolved": "https://registry.npmjs.org/nise/-/nise-5.1.5.tgz", + "integrity": "sha512-VJuPIfUFaXNRzETTQEEItTOP8Y171ijr+JLq42wHes3DiryR8vT+1TXQW/Rx8JNUhyYYWyIvjXTU6dOhJcs9Nw==", + "dev": true, + "dependencies": { + "@sinonjs/commons": "^2.0.0", + "@sinonjs/fake-timers": "^10.0.2", + "@sinonjs/text-encoding": "^0.7.1", + "just-extend": "^4.0.2", + "path-to-regexp": "^1.7.0" + } + }, + "node_modules/nise/node_modules/@sinonjs/commons": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-2.0.0.tgz", + "integrity": "sha512-uLa0j859mMrg2slwQYdO/AkrOfmH+X6LTVmNTS9CqexuE2IvVORIkSpJLqePAbEnKJ77aMmCwr1NUZ57120Xcg==", + "dev": true, + "dependencies": { + "type-detect": "4.0.8" + } + }, + "node_modules/nise/node_modules/@sinonjs/fake-timers": { + "version": "10.3.0", + "resolved": "https://registry.npmjs.org/@sinonjs/fake-timers/-/fake-timers-10.3.0.tgz", + "integrity": "sha512-V4BG07kuYSUkTCSBHG8G8TNhM+F19jXFWnQtzj+we8DrkpSBCee9Z3Ms8yiGer/dlmhe35/Xdgyo3/0rQKg7YA==", + "dev": true, + "dependencies": { + "@sinonjs/commons": "^3.0.0" + } + }, + "node_modules/nise/node_modules/@sinonjs/fake-timers/node_modules/@sinonjs/commons": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-3.0.0.tgz", + "integrity": "sha512-jXBtWAF4vmdNmZgD5FoKsVLv3rPgDnLgPbU84LIJ3otV44vJlDRokVng5v8NFJdCf/da9legHcKaRuZs4L7faA==", + "dev": true, + "dependencies": { + "type-detect": "4.0.8" + } + }, "node_modules/nock": { "version": "13.3.8", "resolved": "https://registry.npmjs.org/nock/-/nock-13.3.8.tgz", @@ -9602,6 +9698,21 @@ "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==", "dev": true }, + "node_modules/path-to-regexp": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-1.8.0.tgz", + "integrity": "sha512-n43JRhlUKUAlibEJhPeir1ncUID16QnEjNpwzNdO3Lm4ywrBpBZ5oLD0I6br9evr1Y9JTqwRtAh7JLoOzAQdVA==", + "dev": true, + "dependencies": { + "isarray": "0.0.1" + } + }, + "node_modules/path-to-regexp/node_modules/isarray": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz", + "integrity": "sha512-D2S+3GLxWH+uhrNEcoh/fnmYeP8E8/zHl644d/jdA0g2uyXvy3sb0qxotE+ne0LtccHknQzWwZEzhak7oJ0COQ==", + "dev": true + }, "node_modules/path-type": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/path-type/-/path-type-4.0.0.tgz", @@ -10910,6 +11021,33 @@ "node": ">=4" } }, + "node_modules/sinon": { + "version": "17.0.1", + "resolved": "https://registry.npmjs.org/sinon/-/sinon-17.0.1.tgz", + "integrity": "sha512-wmwE19Lie0MLT+ZYNpDymasPHUKTaZHUH/pKEubRXIzySv9Atnlw+BUMGCzWgV7b7wO+Hw6f1TEOr0IUnmU8/g==", + "dev": true, + "dependencies": { + "@sinonjs/commons": "^3.0.0", + "@sinonjs/fake-timers": "^11.2.2", + "@sinonjs/samsam": "^8.0.0", + "diff": "^5.1.0", + "nise": "^5.1.5", + "supports-color": "^7.2.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/sinon" + } + }, + "node_modules/sinon/node_modules/diff": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/diff/-/diff-5.1.0.tgz", + "integrity": "sha512-D+mk+qE8VC/PAUrlAU34N+VfXev0ghe5ywmpqrawphmVZc1bEfn56uo9qpyGp1p4xpzOHkSW4ztBd6L7Xx4ACw==", + "dev": true, + "engines": { + "node": ">=0.3.1" + } + }, "node_modules/slash": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/slash/-/slash-3.0.0.tgz", @@ -12101,18 +12239,49 @@ } }, "packages/spacecat-shared-data-access": { + "name": "@adobe/spacecat-shared-data-access", "version": "1.1.0", "license": "Apache-2.0", "dependencies": { - "@adobe/spacecat-shared-utils": "1.1.0" + "@adobe/spacecat-shared-dynamo": "1.1.2", + "@adobe/spacecat-shared-utils": "1.1.0", + "uuid": "9.0.1" }, "devDependencies": { - "chai": "4.3.10" + "chai": "4.3.10", + "sinon": "^17.0.1" + } + }, + "packages/spacecat-shared-data-access/node_modules/@adobe/spacecat-shared-dynamo": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@adobe/spacecat-shared-dynamo/-/spacecat-shared-dynamo-1.1.2.tgz", + "integrity": "sha512-xR1iFHhDBOhQ2ci0qH6dzz954+KJpSPoHaUynTphLNTVNlToXHkQiSE8X+z8cX9G5NCm9HG3tgNYv/9wRK6zPQ==", + "dependencies": { + "@adobe/spacecat-shared-utils": "1.0.1", + "@aws-sdk/client-dynamodb": "3.454.0", + "@aws-sdk/lib-dynamodb": "3.454.0" + } + }, + "packages/spacecat-shared-data-access/node_modules/@adobe/spacecat-shared-dynamo/node_modules/@adobe/spacecat-shared-utils": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@adobe/spacecat-shared-utils/-/spacecat-shared-utils-1.0.1.tgz", + "integrity": "sha512-6HiCywan5y9jpbFYkQX87Vg0q5dXoJf2m6MiDvNBEJdREdUxFwE+oYA6Fr86qqd32ECnCeYCFO13HcNRn1RxkQ==" + }, + "packages/spacecat-shared-data-access/node_modules/uuid": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", + "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==", + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "bin": { + "uuid": "dist/bin/uuid" } }, "packages/spacecat-shared-dynamo": { "name": "@adobe/spacecat-shared-dynamo", - "version": "1.1.0", + "version": "1.1.2", "license": "Apache-2.0", "dependencies": { "@adobe/spacecat-shared-utils": "1.1.0", diff --git a/packages/spacecat-shared-data-access/docs/schema.json b/packages/spacecat-shared-data-access/docs/schema.json new file mode 100644 index 000000000..4ba87ec92 --- /dev/null +++ b/packages/spacecat-shared-data-access/docs/schema.json @@ -0,0 +1,290 @@ +{ + "ModelName": "StarCatalogue", + "ModelMetadata": { + "Author": "Dominique Jäggi", + "DateCreated": "Nov 23, 2023, 07:00 AM", + "DateLastModified": "Nov 23, 2023, 07:00 AM", + "Description": "", + "AWSService": "Amazon DynamoDB", + "Version": "3.0" + }, + "DataModel": [ + { + "TableName": "sites", + "KeyAttributes": { + "PartitionKey": { + "AttributeName": "id", + "AttributeType": "S" + } + }, + "NonKeyAttributes": [ + { + "AttributeName": "baseURL", + "AttributeType": "S" + }, + { + "AttributeName": "imsOrgId", + "AttributeType": "S" + }, + { + "AttributeName": "GSI1PK", + "AttributeType": "S" + }, + { + "AttributeName": "createdAt", + "AttributeType": "S" + }, + { + "AttributeName": "updatedAt", + "AttributeType": "S" + } + ], + "GlobalSecondaryIndexes": [ + { + "IndexName": "all_sites", + "KeyAttributes": { + "PartitionKey": { + "AttributeName": "GSI1PK", + "AttributeType": "S" + }, + "SortKey": { + "AttributeName": "baseURL", + "AttributeType": "S" + } + }, + "Projection": { + "ProjectionType": "ALL" + } + } + ], + "DataAccess": { + "MySql": {} + }, + "SampleDataFormats": { + "id": [ + "identifiers", + "UUID" + ], + "baseURL": [ + "identifiers", + "URL" + ], + "imsOrgId": [ + "identifiers", + "UUID" + ] + }, + "BillingMode": "PROVISIONED", + "ProvisionedCapacitySettings": { + "ProvisionedThroughput": { + "ReadCapacityUnits": 5, + "WriteCapacityUnits": 5 + }, + "AutoScalingRead": { + "ScalableTargetRequest": { + "MinCapacity": 1, + "MaxCapacity": 10, + "ServiceRole": "AWSServiceRoleForApplicationAutoScaling_DynamoDBTable" + }, + "ScalingPolicyConfiguration": { + "TargetValue": 70 + } + }, + "AutoScalingWrite": { + "ScalableTargetRequest": { + "MinCapacity": 1, + "MaxCapacity": 10, + "ServiceRole": "AWSServiceRoleForApplicationAutoScaling_DynamoDBTable" + }, + "ScalingPolicyConfiguration": { + "TargetValue": 70 + } + } + } + }, + { + "TableName": "audits", + "KeyAttributes": { + "PartitionKey": { + "AttributeName": "siteId", + "AttributeType": "S" + }, + "SortKey": { + "AttributeName": "SK", + "AttributeType": "S" + } + }, + "NonKeyAttributes": [ + { + "AttributeName": "auditedAt", + "AttributeType": "S" + }, + { + "AttributeName": "auditResult", + "AttributeType": "M" + }, + { + "AttributeName": "auditType", + "AttributeType": "S" + }, + { + "AttributeName": "expiresAt", + "AttributeType": "N" + }, + { + "AttributeName": "fullAuditRef", + "AttributeType": "S" + } + ], + "DataAccess": { + "MySql": {} + }, + "SampleDataFormats": { + "siteId": [ + "identifiers", + "UUID" + ], + "auditedAt": [ + "date", + "ISO 8601 date and time" + ], + "fullAuditRef": [ + "identifiers", + "URL" + ] + }, + "BillingMode": "PROVISIONED", + "ProvisionedCapacitySettings": { + "ProvisionedThroughput": { + "ReadCapacityUnits": 5, + "WriteCapacityUnits": 5 + }, + "AutoScalingRead": { + "ScalableTargetRequest": { + "MinCapacity": 1, + "MaxCapacity": 10, + "ServiceRole": "AWSServiceRoleForApplicationAutoScaling_DynamoDBTable" + }, + "ScalingPolicyConfiguration": { + "TargetValue": 70 + } + }, + "AutoScalingWrite": { + "ScalableTargetRequest": { + "MinCapacity": 1, + "MaxCapacity": 10, + "ServiceRole": "AWSServiceRoleForApplicationAutoScaling_DynamoDBTable" + }, + "ScalingPolicyConfiguration": { + "TargetValue": 70 + } + } + } + }, + { + "TableName": "latest_audits", + "KeyAttributes": { + "PartitionKey": { + "AttributeName": "siteId", + "AttributeType": "S" + }, + "SortKey": { + "AttributeName": "SK", + "AttributeType": "S" + } + }, + "NonKeyAttributes": [ + { + "AttributeName": "auditedAt", + "AttributeType": "S" + }, + { + "AttributeName": "auditResult", + "AttributeType": "M" + }, + { + "AttributeName": "auditType", + "AttributeType": "S" + }, + { + "AttributeName": "expiresAt", + "AttributeType": "N" + }, + { + "AttributeName": "fullAuditRef", + "AttributeType": "S" + }, + { + "AttributeName": "GSI1PK", + "AttributeType": "S" + }, + { + "AttributeName": "GSI1SK", + "AttributeType": "S" + } + ], + "GlobalSecondaryIndexes": [ + { + "IndexName": "all_latest_audit_scores", + "KeyAttributes": { + "PartitionKey": { + "AttributeName": "GSI1PK", + "AttributeType": "S" + }, + "SortKey": { + "AttributeName": "GSI1SK", + "AttributeType": "S" + } + }, + "Projection": { + "ProjectionType": "ALL" + } + } + ], + "DataAccess": { + "MySql": {} + }, + "SampleDataFormats": { + "siteId": [ + "identifiers", + "UUID" + ], + "auditedAt": [ + "date", + "ISO 8601 date and time" + ], + "fullAuditRef": [ + "identifiers", + "URL" + ] + }, + "BillingMode": "PROVISIONED", + "ProvisionedCapacitySettings": { + "ProvisionedThroughput": { + "ReadCapacityUnits": 5, + "WriteCapacityUnits": 5 + }, + "AutoScalingRead": { + "ScalableTargetRequest": { + "MinCapacity": 1, + "MaxCapacity": 10, + "ServiceRole": "AWSServiceRoleForApplicationAutoScaling_DynamoDBTable" + }, + "ScalingPolicyConfiguration": { + "TargetValue": 70 + } + }, + "AutoScalingWrite": { + "ScalableTargetRequest": { + "MinCapacity": 1, + "MaxCapacity": 10, + "ServiceRole": "AWSServiceRoleForApplicationAutoScaling_DynamoDBTable" + }, + "ScalingPolicyConfiguration": { + "TargetValue": 70 + } + } + } + } + ] +} diff --git a/packages/spacecat-shared-data-access/package.json b/packages/spacecat-shared-data-access/package.json index 028e9cff9..04ff67eb5 100644 --- a/packages/spacecat-shared-data-access/package.json +++ b/packages/spacecat-shared-data-access/package.json @@ -29,9 +29,12 @@ "access": "public" }, "dependencies": { - "@adobe/spacecat-shared-utils": "1.1.0" + "@adobe/spacecat-shared-dynamo": "1.1.2", + "@adobe/spacecat-shared-utils": "1.1.0", + "uuid": "9.0.1" }, "devDependencies": { - "chai": "4.3.10" + "chai": "4.3.10", + "sinon": "17.0.1" } } diff --git a/packages/spacecat-shared-data-access/src/audits/accessPatterns.js b/packages/spacecat-shared-data-access/src/audits/accessPatterns.js new file mode 100644 index 000000000..3be1877a2 --- /dev/null +++ b/packages/spacecat-shared-data-access/src/audits/accessPatterns.js @@ -0,0 +1,95 @@ +/* + * Copyright 2023 Adobe. All rights reserved. + * This file is licensed to you under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS + * OF ANY KIND, either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ + +/** + * Retrieves audits for a specified site. If an audit type is provided, + * it returns only audits of that type. + * + * @param {DynamoDbClient} dynamoClient - The DynamoDB client. + * @param {Logger} log - The logger. + * @param {string} siteId - The ID of the site for which audits are being retrieved. + * @param {string} [auditType] - Optional. The type of audits to retrieve. + * @returns {Promise} A promise that resolves to an array of audits for the specified site. + */ +export const getAuditsForSite = async (dynamoClient, log, siteId, auditType) => { + // Base query parameters + const queryParams = { + TableName: 'audits', + KeyConditionExpression: 'siteId = :siteId', + ExpressionAttributeValues: { + ':siteId': siteId, + }, + }; + + if (auditType !== undefined) { + queryParams.KeyConditionExpression += ' AND begins_with(SK, :auditType)'; + queryParams.ExpressionAttributeValues[':auditType'] = `${auditType}#`; + } + + return dynamoClient.query(queryParams); +}; + +/** + * Retrieves the latest audits of a specific type across all sites. + * + * @param {DynamoDbClient} dynamoClient - The DynamoDB client. + * @param {Logger} log - The logger. + * @param {string} auditType - The type of audits to retrieve. + * @param {boolean} ascending - Determines if the audits should be sorted ascending + * or descending by scores. + * @returns {Promise} A promise that resolves to an array of the latest + * audits of the specified type. + */ +export const getLatestAudits = async ( + dynamoClient, + log, + auditType, + ascending = true, +) => dynamoClient.query({ + TableName: 'latest_audits', + IndexName: 'all_latest_audit_scores', + KeyConditionExpression: 'GSI1PK = :gsi1pk AND begins_with(GSI1SK, :auditType)', + ExpressionAttributeValues: { + ':gsi1pk': 'ALL_LATEST_AUDITS', + ':auditType': `${auditType}#`, + }, + ScanIndexForward: ascending, // Sorts ascending if true, descending if false +}); + +/** + * Retrieves the latest audit for a specified site and audit type. + * + * @param {DynamoDbClient} dynamoClient - The DynamoDB client. + * @param {Logger} log - The logger. + * @param {string} siteId - The ID of the site for which the latest audit is being retrieved. + * @param {string} auditType - The type of audit to retrieve the latest instance of. + * @returns {Promise} A promise that resolves to the latest audit of the + * specified type for the site, or null if none is found. + */ +export const getLatestAuditForSite = async ( + dynamoClient, + log, + siteId, + auditType, +) => { + const latestAudit = await dynamoClient.query({ + TableName: 'latest_audits', + KeyConditionExpression: 'siteId = :siteId AND begins_with(SK, :auditType)', + ExpressionAttributeValues: { + ':siteId': siteId, + ':auditType': `${auditType}#`, + }, + Limit: 1, + }); + + return latestAudit.length > 0 ? latestAudit[0] : null; +}; diff --git a/packages/spacecat-shared-data-access/src/audits/index.js b/packages/spacecat-shared-data-access/src/audits/index.js new file mode 100644 index 000000000..3230ce4f0 --- /dev/null +++ b/packages/spacecat-shared-data-access/src/audits/index.js @@ -0,0 +1,34 @@ +/* + * Copyright 2023 Adobe. All rights reserved. + * This file is licensed to you under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS + * OF ANY KIND, either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ + +import { getAuditsForSite, getLatestAuditForSite, getLatestAudits } from './accessPatterns.js'; + +export const auditFunctions = (dynamoClient, log) => ({ + getAuditsForSite: (siteId, auditType) => getAuditsForSite( + dynamoClient, + log, + siteId, + auditType, + ), + getLatestAudits: (auditType, ascending) => getLatestAudits( + dynamoClient, + log, + auditType, + ascending, + ), + getLatestAuditForSite: (siteId, auditType) => getLatestAuditForSite( + dynamoClient, + log, + siteId, + auditType, + ), +}); diff --git a/packages/spacecat-shared-data-access/src/index.d.ts b/packages/spacecat-shared-data-access/src/index.d.ts new file mode 100644 index 000000000..2605daff7 --- /dev/null +++ b/packages/spacecat-shared-data-access/src/index.d.ts @@ -0,0 +1,71 @@ +/* + * Copyright 2023 Adobe. All rights reserved. + * This file is licensed to you under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS + * OF ANY KIND, either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ + +export interface Site { + id: string; + baseURL: string; + imsOrgId: string; + createdAt: string; + updatedAt: string; +} + +export interface Audit { + siteId: string; + auditedAt: string; + auditResult: object; + auditType: string; + expiresAt: number; + fullAuditRef: string; + createdAt: string; + updatedAt: string; +} + +export interface DataAccess { + getAuditsForSite: ( + siteId: string, + auditType?: string + ) => Promise; + getLatestAuditForSite: ( + siteId: string, + auditType: string, + ) => Promise; + getLatestAudits: ( + auditType: string, + ascending?: boolean, + ) => Promise; + getSites: () => Promise; + getSitesToAudit: () => Promise; + getSitesWithLatestAudit: ( + auditType: string, + sortAuditsAscending?: boolean, + ) => Promise; + getSiteByBaseURL: ( + baseUrl: string, + ) => Promise; + getSiteByBaseURLWithAuditInfo: ( + baseUrl: string, + auditType: string, + latestOnly?: boolean, + ) => Promise; + getSiteByBaseURLWithAudits: ( + baseUrl: string, + auditType: string, + ) => Promise; + getSiteByBaseURLWithLatestAudit: ( + baseUrl: string, + auditType: string, + ) => Promise; +} + +export function createDataAccess( + logger: object, +): DataAccess; diff --git a/packages/spacecat-shared-data-access/src/index.js b/packages/spacecat-shared-data-access/src/index.js new file mode 100644 index 000000000..6f0b9bddf --- /dev/null +++ b/packages/spacecat-shared-data-access/src/index.js @@ -0,0 +1,33 @@ +/* + * Copyright 2023 Adobe. All rights reserved. + * This file is licensed to you under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS + * OF ANY KIND, either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ + +import { createClient } from '@adobe/spacecat-shared-dynamo'; +import { auditFunctions } from './audits/index.js'; +import { siteFunctions } from './sites/index.js'; + +/** + * Creates a data access object. + * + * @param {Logger} log logger + * @returns {object} data access object + */ +export const createDataAccess = (log = console) => { + const dynamoClient = createClient(log); + + const auditFuncs = auditFunctions(dynamoClient, log); + const siteFuncs = siteFunctions(dynamoClient, log); + + return { + ...auditFuncs, + ...siteFuncs, + }; +}; diff --git a/packages/spacecat-shared-data-access/src/models/audit.js b/packages/spacecat-shared-data-access/src/models/audit.js new file mode 100644 index 000000000..76d7c37f5 --- /dev/null +++ b/packages/spacecat-shared-data-access/src/models/audit.js @@ -0,0 +1,60 @@ +/* + * Copyright 2023 Adobe. All rights reserved. + * This file is licensed to you under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS + * OF ANY KIND, either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ + +import { hasText, isIsoDate, isObject } from '@adobe/spacecat-shared-utils'; +import { Base } from './base.js'; + +const EXPIRES_IN_DAYS = 30; + +const Audit = (data = {}) => { + const self = Base(data); + + self.getSiteId = () => self.state.siteId; + self.getAuditedAt = () => self.state.auditedAt; + self.getAuditResult = () => self.state.auditResult; + self.getAuditType = () => self.state.auditType.toLowerCase(); + self.getExpiresAt = () => self.state.expiresAt; + self.getFullAuditRef = () => self.state.fullAuditRef; + + return Object.freeze(self); +}; + +export const createAudit = (data) => { + const newState = { ...data }; + + if (!hasText(newState.siteId)) { + throw new Error('Site ID must be provided'); + } + + if (!isIsoDate(newState.auditedAt)) { + throw new Error('Audited at must be a valid ISO date'); + } + + if (!hasText(newState.auditType)) { + throw new Error('Audit type must be provided'); + } + + if (!isObject(newState.auditResult)) { + throw new Error('Audit result must be an object'); + } + + if (!hasText(newState.fullAuditRef)) { + throw new Error('Full audit ref must be provided'); + } + + if (!newState.expiresAt) { + newState.expiresAt = new Date(newState.auditedAt); + newState.expiresAt.setDate(newState.expiresAt.getDate() + EXPIRES_IN_DAYS); + } + + return Audit(newState); +}; diff --git a/packages/spacecat-shared-data-access/src/models/base.js b/packages/spacecat-shared-data-access/src/models/base.js new file mode 100644 index 000000000..797fa26fa --- /dev/null +++ b/packages/spacecat-shared-data-access/src/models/base.js @@ -0,0 +1,30 @@ +/* + * Copyright 2023 Adobe. All rights reserved. + * This file is licensed to you under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS + * OF ANY KIND, either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ + +import { v4 as uuidv4 } from 'uuid'; +import { isString } from '@adobe/spacecat-shared-utils'; + +export const Base = (data = {}) => { + const self = { state: { ...data } }; + + const newRecord = !isString(self.state.id); + + if (newRecord) { + self.state.id = uuidv4(); + } + + self.getId = () => self.state.id; + self.getCreatedAt = () => self.state.createdAt; + self.getUpdatedAt = () => self.state.updatedAt; + + return self; +}; diff --git a/packages/spacecat-shared-data-access/src/models/site.js b/packages/spacecat-shared-data-access/src/models/site.js new file mode 100644 index 000000000..b08587d05 --- /dev/null +++ b/packages/spacecat-shared-data-access/src/models/site.js @@ -0,0 +1,51 @@ +/* + * Copyright 2023 Adobe. All rights reserved. + * This file is licensed to you under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS + * OF ANY KIND, either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ + +import { hasText, isValidUrl } from '@adobe/spacecat-shared-utils'; +import { Base } from './base.js'; + +const Site = (data = {}) => { + const self = Base(data); + + self.getBaseURL = () => self.state.baseURL; + self.getImsOrgId = () => self.state.imsOrgId; + + self.updateBaseURL = (baseURL) => { + if (!isValidUrl(baseURL)) { + throw new Error('Base URL must be a valid URL'); + } + + self.state.baseURL = baseURL; + return self; + }; + + self.updateImsOrgId = (imsOrgId) => { + if (!hasText(imsOrgId)) { + throw new Error('IMS Org ID must be provided'); + } + + self.state.imsOrgId = imsOrgId; + return self; + }; + + return Object.freeze(self); +}; + +export const createSite = (data) => { + const newState = { ...data }; + + if (!isValidUrl(newState.baseURL)) { + throw new Error('Base URL must be a valid URL'); + } + + return Site(newState); +}; diff --git a/packages/spacecat-shared-data-access/src/sites/accessPatterns.js b/packages/spacecat-shared-data-access/src/sites/accessPatterns.js new file mode 100644 index 000000000..ca862e493 --- /dev/null +++ b/packages/spacecat-shared-data-access/src/sites/accessPatterns.js @@ -0,0 +1,170 @@ +/* + * Copyright 2023 Adobe. All rights reserved. + * This file is licensed to you under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS + * OF ANY KIND, either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ + +import { + getAuditsForSite, + getLatestAuditForSite, + getLatestAudits, +} from '../audits/accessPatterns.js'; + +/** + * Retrieves all sites. + * + * @param {DynamoDbClient} dynamoClient - The DynamoDB client. + * @returns {Promise} A promise that resolves to an array of all sites. + */ +export const getSites = async (dynamoClient) => dynamoClient.query({ + TableName: 'sites', + IndexName: 'all_sites', // GSI name + KeyConditionExpression: 'GSI1PK = :gsi1pk', + ExpressionAttributeValues: { + ':gsi1pk': 'ALL_SITES', + }, +}); + +/** + * Retrieves a list of base URLs for all sites. + * + * @param {DynamoDbClient} dynamoClient - The DynamoDB client. + * @param {Logger} log - The logger. + * @returns {Promise>} A promise that resolves to an array of base URLs for all sites. + */ +export const getSitesToAudit = async (dynamoClient, log) => { + const sites = await getSites(dynamoClient, log); + + return sites.map((item) => item.baseURL); +}; + +/** + * Retrieves sites with their latest audit of a specified type. + * + * @param {DynamoDbClient} dynamoClient - The DynamoDB client. + * @param {Logger} log - The logger. + * @param {string} auditType - The type of the latest audits to retrieve for each site. + * @param {boolean} [sortAuditsAscending] - Optional. Determines if the audits + * should be sorted ascending or descending by scores. + * @returns {Promise} A promise that resolves to an array of site objects, + * each with its latest audit of the specified type. + */ +export const getSitesWithLatestAudit = async ( + dynamoClient, + log, + auditType, + sortAuditsAscending = true, +) => { + const [sites, latestAudits] = await Promise.all([ + getSites(dynamoClient), + getLatestAudits(dynamoClient, log, auditType, sortAuditsAscending), + ]); + + const sitesMap = new Map(sites.map((site) => [site.id, site])); + + return latestAudits.reduce((result, audit) => { + const site = sitesMap.get(audit.siteId); + if (site) { + site.audits = [audit]; + result.push(site); + } + return result; + }, []); +}; + +/** + * Retrieves a site by its base URL. + * + * @param {DynamoDbClient} dynamoClient - The DynamoDB client. + * @param {Logger} log - The logger. + * @param {string} baseUrl - The base URL of the site to retrieve. + * @returns {Promise} A promise that resolves to the site object if found, + * otherwise null. + */ +export const getSiteByBaseURL = async ( + dynamoClient, + log, + baseUrl, +) => dynamoClient.getItem('sites', { + GSI1PK: 'ALL_SITES', + baseUrl, +}); + +/** + * Retrieves a site by its base URL, along with associated audit information. + * + * @param {DynamoDbClient} dynamoClient - The DynamoDB client. + * @param {Logger} log - The logger. + * @param {string} baseUrl - The base URL of the site to retrieve. + * @param {string} auditType - The type of audits to retrieve for the site. + * @param {boolean} [latestOnly=false] - Determines if only the latest audit should be retrieved. + * @returns {Promise} A promise that resolves to the site object with audit + * data if found, otherwise null. + */ +export const getSiteByBaseURLWithAuditInfo = async ( + dynamoClient, + log, + baseUrl, + auditType, + latestOnly = false, +) => { + const site = await getSiteByBaseURL(dynamoClient, log, baseUrl); + + if (!site) { + return null; + } + + site.audits = latestOnly + ? [await getLatestAuditForSite( + dynamoClient, + log, + site.id, + auditType, + )].filter((audit) => audit != null) + : await getAuditsForSite( + dynamoClient, + log, + site.id, + auditType, + ); + + return site; +}; + +/** + * Retrieves a site by its base URL, including all its audits. + * + * @param {DynamoDbClient} dynamoClient - The DynamoDB client. + * @param {Logger} log - The logger. + * @param {string} baseUrl - The base URL of the site to retrieve. + * @param {string} auditType - The type of audits to retrieve for the site. + * @returns {Promise} A promise that resolves to the site object with all its audits. + */ +export const getSiteByBaseURLWithAudits = async ( + dynamoClient, + log, + baseUrl, + auditType, +) => getSiteByBaseURLWithAuditInfo(dynamoClient, log, baseUrl, auditType, false); + +/** + * Retrieves a site by its base URL, including only its latest audit. + * + * @param {DynamoDbClient} dynamoClient - The DynamoDB client. + * @param {Logger} log - The logger. + * @param {string} baseUrl - The base URL of the site to retrieve. + * @param {string} auditType - The type of the latest audit to retrieve for the site. + * @returns {Promise} A promise that resolves to the site object with its latest audit. + */ +export const getSiteByBaseURLWithLatestAudit = async ( + dynamoClient, + log, + baseUrl, + auditType, +) => getSiteByBaseURLWithAuditInfo(dynamoClient, log, baseUrl, auditType, true); diff --git a/packages/spacecat-shared-data-access/src/sites/index.js b/packages/spacecat-shared-data-access/src/sites/index.js new file mode 100644 index 000000000..e9ac693bd --- /dev/null +++ b/packages/spacecat-shared-data-access/src/sites/index.js @@ -0,0 +1,59 @@ +/* + * Copyright 2023 Adobe. All rights reserved. + * This file is licensed to you under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS + * OF ANY KIND, either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ + +import { + getSiteByBaseURL, + getSiteByBaseURLWithAuditInfo, + getSiteByBaseURLWithAudits, + getSiteByBaseURLWithLatestAudit, getSites, getSitesToAudit, getSitesWithLatestAudit, +} from './accessPatterns.js'; + +export const siteFunctions = (dynamoClient, log) => ({ + getSites: () => getSites( + dynamoClient, + log, + ), + getSitesToAudit: () => getSitesToAudit( + dynamoClient, + log, + ), + getSitesWithLatestAudit: (auditType, sortAuditsAscending) => getSitesWithLatestAudit( + dynamoClient, + log, + auditType, + sortAuditsAscending, + ), + getSiteByBaseURL: (baseUrl) => getSiteByBaseURL( + dynamoClient, + log, + baseUrl, + ), + getSiteByBaseURLWithAuditInfo: (baseUrl, auditType, latestOnly) => getSiteByBaseURLWithAuditInfo( + dynamoClient, + log, + baseUrl, + auditType, + latestOnly, + ), + getSiteByBaseURLWithAudits: (baseUrl, auditType) => getSiteByBaseURLWithAudits( + dynamoClient, + log, + baseUrl, + auditType, + ), + getSiteByBaseURLWithLatestAudit: (baseUrl, auditType) => getSiteByBaseURLWithLatestAudit( + dynamoClient, + log, + baseUrl, + auditType, + ), +}); diff --git a/packages/spacecat-shared-data-access/test/audits/index.test.js b/packages/spacecat-shared-data-access/test/audits/index.test.js new file mode 100644 index 000000000..9e8a63db3 --- /dev/null +++ b/packages/spacecat-shared-data-access/test/audits/index.test.js @@ -0,0 +1,79 @@ +/* + * Copyright 2023 Adobe. All rights reserved. + * This file is licensed to you under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS + * OF ANY KIND, either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ + +/* eslint-env mocha */ + +import { expect } from 'chai'; +import sinon from 'sinon'; + +import { auditFunctions } from '../../src/audits/index.js'; + +describe('Audit Index Tests', () => { + describe('Audit Functions Export Tests', () => { + const mockDynamoClient = {}; + const mockLog = {}; + + const exportedFunctions = auditFunctions(mockDynamoClient, mockLog); + + it('exports getAuditsForSite function', () => { + expect(exportedFunctions).to.have.property('getAuditsForSite'); + expect(exportedFunctions.getAuditsForSite).to.be.a('function'); + }); + + it('exports getLatestAudits function', () => { + expect(exportedFunctions).to.have.property('getLatestAudits'); + expect(exportedFunctions.getLatestAudits).to.be.a('function'); + }); + + it('exports getLatestAuditForSite function', () => { + expect(exportedFunctions).to.have.property('getLatestAuditForSite'); + expect(exportedFunctions.getLatestAuditForSite).to.be.a('function'); + }); + }); + + describe('Audit Functions Tests', () => { + let mockDynamoClient; + let mockLog; + let exportedFunctions; + + beforeEach(() => { + mockDynamoClient = { + query: sinon.stub().returns(Promise.resolve([])), + }; + mockLog = { log: sinon.stub() }; + + exportedFunctions = auditFunctions(mockDynamoClient, mockLog); + }); + + it('calls getAuditsForSite and return an array', async () => { + const result = await exportedFunctions.getAuditsForSite('siteId', 'auditType'); + expect(result).to.be.an('array'); + // eslint-disable-next-line no-unused-expressions + expect(mockDynamoClient.query.called).to.be.true; + }); + + it('calls getLatestAudits and return an array', async () => { + const result = await exportedFunctions.getLatestAudits('auditType', true); + expect(result).to.be.an('array'); + // eslint-disable-next-line no-unused-expressions + expect(mockDynamoClient.query.called).to.be.true; + }); + + it('calls getLatestAuditForSite and return an array', async () => { + const result = await exportedFunctions.getLatestAuditForSite('siteId', 'auditType'); + // eslint-disable-next-line no-unused-expressions + expect(result).to.be.null; + // eslint-disable-next-line no-unused-expressions + expect(mockDynamoClient.query.called).to.be.true; + }); + }); +}); diff --git a/packages/spacecat-shared-data-access/test/index.test.js b/packages/spacecat-shared-data-access/test/index.test.js new file mode 100644 index 000000000..656b22525 --- /dev/null +++ b/packages/spacecat-shared-data-access/test/index.test.js @@ -0,0 +1,58 @@ +/* + * Copyright 2023 Adobe. All rights reserved. + * This file is licensed to you under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS + * OF ANY KIND, either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ + +/* eslint-env mocha */ + +import { expect } from 'chai'; +import { createDataAccess } from '../src/index.js'; + +describe('Data Access Object Tests', () => { + const auditFunctions = [ + 'getAuditsForSite', + 'getLatestAudits', + 'getLatestAuditForSite', + ]; + const siteFunctions = [ + 'getSites', + 'getSitesToAudit', + 'getSitesWithLatestAudit', + 'getSiteByBaseURL', + 'getSiteByBaseURLWithAuditInfo', + 'getSiteByBaseURLWithAudits', + 'getSiteByBaseURLWithLatestAudit', + ]; + + let dao; + + before(() => { + dao = createDataAccess(); + }); + + it('contains all known audit functions', () => { + auditFunctions.forEach((funcName) => { + expect(dao).to.have.property(funcName); + }); + }); + + it('contains all known site functions', () => { + siteFunctions.forEach((funcName) => { + expect(dao).to.have.property(funcName); + }); + }); + + it('does not contain any unexpected functions', () => { + const expectedFunctions = new Set([...auditFunctions, ...siteFunctions]); + Object.keys(dao).forEach((funcName) => { + expect(expectedFunctions).to.include(funcName); + }); + }); +}); diff --git a/packages/spacecat-shared-data-access/test/models/audit.test.js b/packages/spacecat-shared-data-access/test/models/audit.test.js new file mode 100644 index 000000000..f6d48683e --- /dev/null +++ b/packages/spacecat-shared-data-access/test/models/audit.test.js @@ -0,0 +1,69 @@ +/* + * Copyright 2023 Adobe. All rights reserved. + * This file is licensed to you under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS + * OF ANY KIND, either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ + +/* eslint-env mocha */ + +import { expect } from 'chai'; +import { createAudit } from '../../src/models/audit.js'; + +// Constants for testing +const validData = { + siteId: '123', + auditedAt: new Date().toISOString(), + auditType: 'Type', + auditResult: {}, + fullAuditRef: 'ref123', +}; + +describe('Audit Module Tests', () => { + describe('Validation Tests', () => { + it('throws an error if siteId is not provided', () => { + expect(() => createAudit({ ...validData, siteId: '' })).to.throw('Site ID must be provided'); + }); + + it('throws an error if auditedAt is not a valid ISO date', () => { + expect(() => createAudit({ ...validData, auditedAt: 'invalid-date' })).to.throw('Audited at must be a valid ISO date'); + }); + + it('throws an error if auditType is not provided', () => { + expect(() => createAudit({ ...validData, auditType: '' })).to.throw('Audit type must be provided'); + }); + + it('throws an error if auditResult is not an object', () => { + expect(() => createAudit({ ...validData, auditResult: 'not-an-object' })).to.throw('Audit result must be an object'); + }); + + it('throws an error if fullAuditRef is not provided', () => { + expect(() => createAudit({ ...validData, fullAuditRef: '' })).to.throw('Full audit ref must be provided'); + }); + }); + + describe('Functionality Tests', () => { + it('creates an audit object with correct properties', () => { + const audit = createAudit(validData); + expect(audit).to.be.an('object'); + expect(audit.getSiteId()).to.equal(validData.siteId); + expect(audit.getAuditedAt()).to.equal(validData.auditedAt); + expect(audit.getAuditType()).to.equal(validData.auditType.toLowerCase()); + expect(audit.getAuditResult()).to.deep.equal(validData.auditResult); + expect(audit.getFullAuditRef()).to.equal(validData.fullAuditRef); + }); + + it('automatically sets expiresAt if not provided', () => { + const audit = createAudit(validData); + expect(audit.getExpiresAt()).to.be.a('Date'); + const expectedDate = new Date(validData.auditedAt); + expectedDate.setDate(expectedDate.getDate() + 30); + expect(audit.getExpiresAt().toDateString()).to.equal(expectedDate.toDateString()); + }); + }); +}); diff --git a/packages/spacecat-shared-data-access/test/models/base.test.js b/packages/spacecat-shared-data-access/test/models/base.test.js new file mode 100644 index 000000000..e9264aad9 --- /dev/null +++ b/packages/spacecat-shared-data-access/test/models/base.test.js @@ -0,0 +1,57 @@ +/* + * Copyright 2023 Adobe. All rights reserved. + * This file is licensed to you under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS + * OF ANY KIND, either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ + +/* eslint-env mocha */ + +import { expect } from 'chai'; +import { Base } from '../../src/models/base.js'; + +describe('Base Entity Tests', () => { + describe('Initialization Tests', () => { + it('should automatically assign a UUID if no id is provided', () => { + const baseEntity = Base(); + expect(baseEntity.getId()).to.match(/[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}/); + }); + + it('should retain the provided id if one is provided', () => { + const id = 'test-id'; + const baseEntity = Base({ id }); + expect(baseEntity.getId()).to.equal(id); + }); + }); + + describe('Getter Method Tests', () => { + it('should correctly return the createdAt date if provided', () => { + const createdAt = new Date().toISOString(); + const baseEntity = Base({ createdAt }); + expect(baseEntity.getCreatedAt()).to.equal(createdAt); + }); + + it('should return undefined for createdAt if not provided', () => { + const baseEntity = Base(); + // eslint-disable-next-line no-unused-expressions + expect(baseEntity.getCreatedAt()).to.be.undefined; + }); + + it('should correctly return the updatedAt date if provided', () => { + const updatedAt = new Date().toISOString(); + const baseEntity = Base({ updatedAt }); + expect(baseEntity.getUpdatedAt()).to.equal(updatedAt); + }); + + it('should return undefined for updatedAt if not provided', () => { + const baseEntity = Base(); + // eslint-disable-next-line no-unused-expressions + expect(baseEntity.getUpdatedAt()).to.be.undefined; + }); + }); +}); diff --git a/packages/spacecat-shared-data-access/test/models/site.test.js b/packages/spacecat-shared-data-access/test/models/site.test.js new file mode 100644 index 000000000..899c8fadc --- /dev/null +++ b/packages/spacecat-shared-data-access/test/models/site.test.js @@ -0,0 +1,65 @@ +/* + * Copyright 2023 Adobe. All rights reserved. + * This file is licensed to you under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS + * OF ANY KIND, either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ + +/* eslint-env mocha */ + +import { expect } from 'chai'; +import { createSite } from '../../src/models/site.js'; + +// Constants for testing +const validData = { + baseURL: 'https://www.example.com', + imsOrgId: 'org123', +}; + +describe('Site Module Tests', () => { + describe('Validation Tests', () => { + it('throws an error if baseURL is not a valid URL', () => { + expect(() => createSite({ ...validData, baseURL: 'invalid-url' })).to.throw('Base URL must be a valid URL'); + }); + + it('creates a site object with valid baseURL', () => { + const site = createSite({ ...validData }); + expect(site).to.be.an('object'); + expect(site.getBaseURL()).to.equal(validData.baseURL); + }); + }); + + describe('Site Object Functionality', () => { + let site; + + beforeEach(() => { + // Reset site object before each test + site = createSite(validData); + }); + + it('updates baseURL correctly', () => { + const newURL = 'https://www.newexample.com'; + site.updateBaseURL(newURL); + expect(site.getBaseURL()).to.equal(newURL); + }); + + it('throws an error when updating with an invalid baseURL', () => { + expect(() => site.updateBaseURL('invalid-url')).to.throw('Base URL must be a valid URL'); + }); + + it('updates imsOrgId correctly', () => { + const newImsOrgId = 'newOrg123'; + site.updateImsOrgId(newImsOrgId); + expect(site.getImsOrgId()).to.equal(newImsOrgId); + }); + + it('throws an error when updating with an empty imsOrgId', () => { + expect(() => site.updateImsOrgId('')).to.throw('IMS Org ID must be provided'); + }); + }); +}); diff --git a/packages/spacecat-shared-data-access/test/sites/index.test.js b/packages/spacecat-shared-data-access/test/sites/index.test.js new file mode 100644 index 000000000..a3874983b --- /dev/null +++ b/packages/spacecat-shared-data-access/test/sites/index.test.js @@ -0,0 +1,214 @@ +/* + * Copyright 2023 Adobe. All rights reserved. + * This file is licensed to you under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS + * OF ANY KIND, either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ + +/* eslint-env mocha */ + +import { expect } from 'chai'; +import sinon from 'sinon'; + +import { siteFunctions } from '../../src/sites/index.js'; + +describe('Site Index Tests', () => { + describe('Site Functions Export Tests', () => { + const mockDynamoClient = {}; + const mockLog = {}; + + const exportedFunctions = siteFunctions(mockDynamoClient, mockLog); + + it('exports getSites function', () => { + expect(exportedFunctions).to.have.property('getSites'); + expect(exportedFunctions.getSites).to.be.a('function'); + }); + + it('exports getSitesToAudit function', () => { + expect(exportedFunctions).to.have.property('getSitesToAudit'); + expect(exportedFunctions.getSitesToAudit).to.be.a('function'); + }); + + it('exports getSitesWithLatestAudit function', () => { + expect(exportedFunctions).to.have.property('getSitesWithLatestAudit'); + expect(exportedFunctions.getSitesWithLatestAudit).to.be.a('function'); + }); + + it('exports getSiteByBaseURL function', () => { + expect(exportedFunctions).to.have.property('getSiteByBaseURL'); + expect(exportedFunctions.getSiteByBaseURL).to.be.a('function'); + }); + + it('exports getSiteByBaseURLWithAuditInfo function', () => { + expect(exportedFunctions).to.have.property('getSiteByBaseURLWithAuditInfo'); + expect(exportedFunctions.getSiteByBaseURLWithAuditInfo).to.be.a('function'); + }); + + it('exports getSiteByBaseURLWithAudits function', () => { + expect(exportedFunctions).to.have.property('getSiteByBaseURLWithAudits'); + expect(exportedFunctions.getSiteByBaseURLWithAudits).to.be.a('function'); + }); + + it('exports getSiteByBaseURLWithLatestAudit function', () => { + expect(exportedFunctions).to.have.property('getSiteByBaseURLWithLatestAudit'); + expect(exportedFunctions.getSiteByBaseURLWithLatestAudit).to.be.a('function'); + }); + }); + + describe('Site Functions Tests', () => { + let mockDynamoClient; + let mockLog; + let exportedFunctions; + + beforeEach(() => { + mockDynamoClient = { + query: sinon.stub().returns(Promise.resolve([])), + getItem: sinon.stub().returns(Promise.resolve(null)), + }; + mockLog = { log: sinon.stub() }; + + exportedFunctions = siteFunctions(mockDynamoClient, mockLog); + }); + + it('calls getSites and returns an array', async () => { + const result = await exportedFunctions.getSites(); + expect(result).to.be.an('array'); + // eslint-disable-next-line no-unused-expressions + expect(mockDynamoClient.query.called).to.be.true; + }); + + it('calls getSitesToAudit and returns an array', async () => { + const result = await exportedFunctions.getSitesToAudit(); + expect(result).to.be.an('array'); + // eslint-disable-next-line no-unused-expressions + expect(mockDynamoClient.query.called).to.be.true; + }); + + it('calls getSitesWithLatestAudit and returns an array', async () => { + const result = await exportedFunctions.getSitesWithLatestAudit(); + expect(result).to.be.an('array'); + // eslint-disable-next-line no-unused-expressions + expect(mockDynamoClient.query.called).to.be.true; + }); + + it('calls getSitesWithLatestAudit and handles latestAudits', async () => { + const mockSiteData = [{ + id: 'site1', + baseUrl: 'https://example.com', + }]; + + const mockAuditData = [{ + id: 'audit1', + siteId: 'site1', + auditType: 'type1', + }]; + + mockDynamoClient.query.onFirstCall().resolves(mockSiteData); + mockDynamoClient.query.onSecondCall().resolves(mockAuditData); + + const result = await exportedFunctions.getSitesWithLatestAudit('auditType'); + // eslint-disable-next-line no-unused-expressions + expect(result).to.be.an('array').that.has.lengthOf(1); + }); + + it('calls getSitesWithLatestAudit and handles empty latestAudits', async () => { + const mockSiteData = [{ + id: 'site1', + baseUrl: 'https://example.com', + }]; + + const mockAuditData = []; + + mockDynamoClient.query.onFirstCall().resolves(mockSiteData); + mockDynamoClient.query.onSecondCall().resolves(mockAuditData); + + const result = await exportedFunctions.getSitesWithLatestAudit('auditType'); + // eslint-disable-next-line no-unused-expressions + expect(result).to.be.an('array').that.is.empty; + }); + + it('calls getSiteByBaseURL and returns an array/object', async () => { + const result = await exportedFunctions.getSiteByBaseURL(); + // eslint-disable-next-line no-unused-expressions + expect(result).to.be.null; + // eslint-disable-next-line no-unused-expressions + expect(mockDynamoClient.getItem.called).to.be.true; + }); + + it('calls getSiteByBaseURLWithAuditInfo and returns an array/object', async () => { + const result = await exportedFunctions.getSiteByBaseURLWithAuditInfo(); + // eslint-disable-next-line no-unused-expressions + expect(result).to.be.null; + // eslint-disable-next-line no-unused-expressions + expect(mockDynamoClient.getItem.called).to.be.true; + }); + + it('calls getSiteByBaseURLWithAuditInfo and returns null when site is undefined', async () => { + mockDynamoClient.query.resolves(undefined); + + const result = await exportedFunctions.getSiteByBaseURLWithAuditInfo('baseUrl', 'auditType'); + // eslint-disable-next-line no-unused-expressions + expect(result).to.be.null; + }); + + it('calls getSiteByBaseURLWithAuditInfo and assigns latest audit when latestOnly is true', async () => { + const mockSiteData = { + id: 'site1', + baseUrl: 'https://example.com', + }; + + const mockLatestAuditData = [{ + id: 'audit1', + siteId: 'site1', + auditType: 'type1', + }]; + + mockDynamoClient.getItem.onFirstCall().resolves(mockSiteData); + mockDynamoClient.query.onFirstCall().resolves(mockLatestAuditData); + + const result = await exportedFunctions.getSiteByBaseURLWithAuditInfo('https://example.com', 'type1', true); + expect(result).to.have.property('audits').that.is.an('array').with.lengthOf(1); + expect(result.audits[0]).to.deep.equal(mockLatestAuditData[0]); + }); + + it('calls getSiteByBaseURLWithAuditInfo and assigns all audits when latestOnly is false', async () => { + const mockSiteData = { + id: 'site1', + baseUrl: 'https://example.com', + }; + + const mockLatestAuditData = [{ + id: 'audit1', + siteId: 'site1', + auditType: 'type1', + }]; + + mockDynamoClient.getItem.onFirstCall().resolves(mockSiteData); + mockDynamoClient.query.onFirstCall().resolves(mockLatestAuditData); + + const result = await exportedFunctions.getSiteByBaseURLWithAuditInfo('baseUrl', 'auditType', false); + expect(result).to.have.property('audits').that.is.an('array'); + }); + + it('calls getSiteByBaseURLWithAudits and returns an array/object', async () => { + const result = await exportedFunctions.getSiteByBaseURLWithAudits(); + // eslint-disable-next-line no-unused-expressions + expect(result).to.be.null; + // eslint-disable-next-line no-unused-expressions + expect(mockDynamoClient.getItem.called).to.be.true; + }); + + it('calls getSiteByBaseURLWithLatestAudit and returns an array/object', async () => { + const result = await exportedFunctions.getSiteByBaseURLWithLatestAudit(); + // eslint-disable-next-line no-unused-expressions + expect(result).to.be.null; + // eslint-disable-next-line no-unused-expressions + expect(mockDynamoClient.getItem.called).to.be.true; + }); + }); +}); From 25cbba04843d59a965f8a826c5a1b7e2ef21c82e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominique=20J=C3=A4ggi?= Date: Thu, 30 Nov 2023 10:06:13 +0100 Subject: [PATCH 04/28] feat: add data access layer --- .eslintrc.cjs | 6 + package-lock.json | 15 +- .../spacecat-shared-data-access/package.json | 1 + .../src/audits/accessPatterns.js | 96 ++++++++++-- .../src/audits/index.js | 19 ++- .../src/dto/audit.js | 60 ++++++++ .../src/dto/site.js | 49 +++++++ .../src/index.d.ts | 39 +++-- .../src/models/audit.js | 30 ++++ .../src/models/base.js | 6 + .../src/models/site.js | 24 ++- .../src/sites/accessPatterns.js | 123 ++++++++++++---- .../src/sites/index.js | 11 +- .../test/audits/index.test.js | 83 ++++++++++- .../test/index.test.js | 4 + .../test/models/base.test.js | 2 - .../test/models/site.test.js | 1 - .../test/sites/index.test.js | 137 ++++++++++++++---- .../spacecat-shared-dynamo/src/index.d.ts | 9 +- .../src/modules/getItem.js | 2 +- .../src/modules/removeItem.js | 2 +- 21 files changed, 616 insertions(+), 103 deletions(-) create mode 100644 packages/spacecat-shared-data-access/src/dto/audit.js create mode 100644 packages/spacecat-shared-data-access/src/dto/site.js diff --git a/.eslintrc.cjs b/.eslintrc.cjs index 3462360d2..cdc3af607 100644 --- a/.eslintrc.cjs +++ b/.eslintrc.cjs @@ -34,5 +34,11 @@ module.exports = { files: ['*.js', '*.cjs'], rules: {}, }, + { + files: ["*.test.js"], + rules: { + "no-unused-expressions": "off" + } + } ], }; diff --git a/package-lock.json b/package-lock.json index 0e9459ca5..efe9d97a7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2856,6 +2856,18 @@ "node": ">=4" } }, + "node_modules/chai-as-promised": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/chai-as-promised/-/chai-as-promised-7.1.1.tgz", + "integrity": "sha512-azL6xMoi+uxu6z4rhWQ1jbdUhOMhis2PvscD/xjLqNMkv3BPPp2JyyuTHOrf9BOosGpNQ11v6BKv/g57RXbiaA==", + "dev": true, + "dependencies": { + "check-error": "^1.0.2" + }, + "peerDependencies": { + "chai": ">= 2.1.2 < 5" + } + }, "node_modules/chalk": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", @@ -12249,7 +12261,8 @@ }, "devDependencies": { "chai": "4.3.10", - "sinon": "^17.0.1" + "chai-as-promised": "^7.1.1", + "sinon": "17.0.1" } }, "packages/spacecat-shared-data-access/node_modules/@adobe/spacecat-shared-dynamo": { diff --git a/packages/spacecat-shared-data-access/package.json b/packages/spacecat-shared-data-access/package.json index 04ff67eb5..87c38fd66 100644 --- a/packages/spacecat-shared-data-access/package.json +++ b/packages/spacecat-shared-data-access/package.json @@ -35,6 +35,7 @@ }, "devDependencies": { "chai": "4.3.10", + "chai-as-promised": "7.1.1", "sinon": "17.0.1" } } diff --git a/packages/spacecat-shared-data-access/src/audits/accessPatterns.js b/packages/spacecat-shared-data-access/src/audits/accessPatterns.js index 3be1877a2..a3b0ac77d 100644 --- a/packages/spacecat-shared-data-access/src/audits/accessPatterns.js +++ b/packages/spacecat-shared-data-access/src/audits/accessPatterns.js @@ -10,6 +10,11 @@ * governing permissions and limitations under the License. */ +import { isObject } from '@adobe/spacecat-shared-utils'; + +import { AuditDto } from '../dto/audit.js'; +import { createAudit } from '../models/audit.js'; + /** * Retrieves audits for a specified site. If an audit type is provided, * it returns only audits of that type. @@ -35,7 +40,39 @@ export const getAuditsForSite = async (dynamoClient, log, siteId, auditType) => queryParams.ExpressionAttributeValues[':auditType'] = `${auditType}#`; } - return dynamoClient.query(queryParams); + const dynamoItems = await dynamoClient.query(queryParams); + + return dynamoItems.map((item) => AuditDto.fromDynamoItem(item)); +}; + +/** + * Retrieves a specific audit for a specified site. + * + * @param {DynamoDbClient} dynamoClient - The DynamoDB client. + * @param {Logger} log - The logger. + * @param {string} siteId - The ID of the site for which to retrieve the audit. + * @param {string} auditType - The type of audit to retrieve. + * @param auditedAt - The ISO 8601 timestamp of the audit. + * @returns {Promise|null>} + */ +export const getAuditForSite = async ( + dynamoClient, + log, + siteId, + auditType, + auditedAt, +) => { + const audit = await dynamoClient.query({ + TableName: 'audits', + KeyConditionExpression: 'siteId = :siteId AND SK = :sk', + ExpressionAttributeValues: { + ':siteId': siteId, + ':sk': `${auditType}#${auditedAt}}`, + }, + Limit: 1, + }); + + return audit.length > 0 ? AuditDto.fromDynamoItem(audit[0]) : null; }; /** @@ -54,16 +91,20 @@ export const getLatestAudits = async ( log, auditType, ascending = true, -) => dynamoClient.query({ - TableName: 'latest_audits', - IndexName: 'all_latest_audit_scores', - KeyConditionExpression: 'GSI1PK = :gsi1pk AND begins_with(GSI1SK, :auditType)', - ExpressionAttributeValues: { - ':gsi1pk': 'ALL_LATEST_AUDITS', - ':auditType': `${auditType}#`, - }, - ScanIndexForward: ascending, // Sorts ascending if true, descending if false -}); +) => { + const dynamoItems = await dynamoClient.query({ + TableName: 'latest_audits', + IndexName: 'all_latest_audit_scores', + KeyConditionExpression: 'GSI1PK = :gsi1pk AND begins_with(GSI1SK, :auditType)', + ExpressionAttributeValues: { + ':gsi1pk': 'ALL_LATEST_AUDITS', + ':auditType': `${auditType}#`, + }, + ScanIndexForward: ascending, // Sorts ascending if true, descending if false + }); + + return dynamoItems.map((item) => AuditDto.fromDynamoItem(item)); +}; /** * Retrieves the latest audit for a specified site and audit type. @@ -72,7 +113,7 @@ export const getLatestAudits = async ( * @param {Logger} log - The logger. * @param {string} siteId - The ID of the site for which the latest audit is being retrieved. * @param {string} auditType - The type of audit to retrieve the latest instance of. - * @returns {Promise} A promise that resolves to the latest audit of the + * @returns {Promise} A promise that resolves to the latest audit of the * specified type for the site, or null if none is found. */ export const getLatestAuditForSite = async ( @@ -91,5 +132,34 @@ export const getLatestAuditForSite = async ( Limit: 1, }); - return latestAudit.length > 0 ? latestAudit[0] : null; + return latestAudit.length > 0 ? AuditDto.fromDynamoItem(latestAudit[0]) : null; +}; + +/** + * Adds an audit. + * + * @param {DynamoDbClient} dynamoClient - The DynamoDB client. + * @param {Logger} log - The logger. + * @param {object} auditData - The audit data. + * @returns {Promise>} + */ +export const addAudit = async (dynamoClient, log, auditData) => { + const audit = createAudit(auditData); + const existingAudit = await getAuditForSite( + dynamoClient, + log, + audit.getSiteId(), + audit.getAuditType(), + audit.getAuditedAt(), + ); + + if (isObject(existingAudit)) { + throw new Error('Audit already exists'); + } + + // TODO: Add transaction support + await dynamoClient.putItem('audits', AuditDto.toDynamoItem(audit)); + await dynamoClient.putItem('latest_audits', AuditDto.toDynamoItem(audit, true)); + + return audit; }; diff --git a/packages/spacecat-shared-data-access/src/audits/index.js b/packages/spacecat-shared-data-access/src/audits/index.js index 3230ce4f0..87645dde7 100644 --- a/packages/spacecat-shared-data-access/src/audits/index.js +++ b/packages/spacecat-shared-data-access/src/audits/index.js @@ -10,9 +10,21 @@ * governing permissions and limitations under the License. */ -import { getAuditsForSite, getLatestAuditForSite, getLatestAudits } from './accessPatterns.js'; +import { + addAudit, getAuditForSite, + getAuditsForSite, + getLatestAuditForSite, + getLatestAudits, +} from './accessPatterns.js'; export const auditFunctions = (dynamoClient, log) => ({ + getAuditForSite: (siteId, auditType, auditedAt) => getAuditForSite( + dynamoClient, + log, + siteId, + auditType, + auditedAt, + ), getAuditsForSite: (siteId, auditType) => getAuditsForSite( dynamoClient, log, @@ -31,4 +43,9 @@ export const auditFunctions = (dynamoClient, log) => ({ siteId, auditType, ), + addAudit: (auditData) => addAudit( + dynamoClient, + log, + auditData, + ), }); diff --git a/packages/spacecat-shared-data-access/src/dto/audit.js b/packages/spacecat-shared-data-access/src/dto/audit.js new file mode 100644 index 000000000..ba621f439 --- /dev/null +++ b/packages/spacecat-shared-data-access/src/dto/audit.js @@ -0,0 +1,60 @@ +/* + * Copyright 2023 Adobe. All rights reserved. + * This file is licensed to you under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS + * OF ANY KIND, either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ + +import { createAudit } from '../models/audit.js'; + +/** + * Data transfer object for Audit. + */ +export const AuditDto = { + /** + * Converts a Audit object into a DynamoDB item. + * @param {Readonly} audit - Audit object. + * @param {boolean} latestAudit - If true, returns the latest audit flavor. + * @returns {{siteId, auditedAt, auditResult, auditType, expiresAt, fullAuditRef, SK: string}} + */ + toDynamoItem: (audit, latestAudit = false) => { + const latestAuditProps = latestAudit ? { + GSI1PK: 'ALL_LATEST_AUDITS', + GSI1SK: `${audit.getAuditType()}#${Object.values(audit.getScores()).join('#')}`, + } : {}; + + return { + siteId: audit.getSiteId(), + auditedAt: audit.getAuditedAt(), + auditResult: audit.getAuditResult(), + auditType: audit.getAuditType(), + expiresAt: audit.getExpiresAt(), + fullAuditRef: audit.getFullAuditRef(), + SK: `${audit.getAuditType()}#${audit.getAuditedAt()}`, + ...latestAuditProps, + }; + }, + + /** + * Converts a DynamoDB item into a Audit object. + * @param {object } dynamoItem - DynamoDB item. + * @returns {Readonly} Audit object. + */ + fromDynamoItem: (dynamoItem) => { + const auditData = { + siteId: dynamoItem.siteId, + auditedAt: dynamoItem.auditedAt, + auditResult: dynamoItem.auditResult, + auditType: dynamoItem.auditType, + expiresAt: dynamoItem.expiresAt, + fullAuditRef: dynamoItem.fullAuditRef, + }; + + return createAudit(auditData); + }, +}; diff --git a/packages/spacecat-shared-data-access/src/dto/site.js b/packages/spacecat-shared-data-access/src/dto/site.js new file mode 100644 index 000000000..c2203f5dc --- /dev/null +++ b/packages/spacecat-shared-data-access/src/dto/site.js @@ -0,0 +1,49 @@ +/* + * Copyright 2023 Adobe. All rights reserved. + * This file is licensed to you under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS + * OF ANY KIND, either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ + +import { createSite } from '../models/site.js'; + +/** + * Data transfer object for Site. + */ +export const SiteDto = { + /** + * Converts a Site object into a DynamoDB item. + * @param {Readonly} site - Site object. + * @returns {{createdAt, baseURL, GSI1PK: string, id, imsOrgId, updatedAt}} + */ + toDynamoItem: (site) => ({ + id: site.getId(), + baseURL: site.getBaseURL(), + imsOrgId: site.getImsOrgId(), + createdAt: site.getCreatedAt(), + updatedAt: site.getUpdatedAt(), + GSI1PK: 'ALL_SITES', + }), + + /** + * Converts a DynamoDB item into a Site object. + * @param {object } dynamoItem - DynamoDB item. + * @returns {Readonly} Site object. + */ + fromDynamoItem: (dynamoItem) => { + const siteData = { + id: dynamoItem.id, + baseURL: dynamoItem.baseURL, + imsOrgId: dynamoItem.imsOrgId, + createdAt: dynamoItem.createdAt, + updatedAt: dynamoItem.updatedAt, + }; + + return createSite(siteData); + }, +}; diff --git a/packages/spacecat-shared-data-access/src/index.d.ts b/packages/spacecat-shared-data-access/src/index.d.ts index 2605daff7..3aa03b507 100644 --- a/packages/spacecat-shared-data-access/src/index.d.ts +++ b/packages/spacecat-shared-data-access/src/index.d.ts @@ -10,23 +10,26 @@ * governing permissions and limitations under the License. */ -export interface Site { - id: string; - baseURL: string; - imsOrgId: string; - createdAt: string; - updatedAt: string; +export interface Audit { + getSiteId: () => string; + getAuditedAt: () => string; + getAuditResult: () => object; + getAuditType: () => object; + getExpiresAt: () => Date; + getFullAuditRef: () => string; + getScores: () => object; } -export interface Audit { - siteId: string; - auditedAt: string; - auditResult: object; - auditType: string; - expiresAt: number; - fullAuditRef: string; - createdAt: string; - updatedAt: string; +export interface Site { + getId: () => string; + getBaseURL: () => string; + getImsOrgId: () => string; + getCreatedAt: () => string; + getUpdatedAt: () => string; + getAudits: () => Audit[]; + updateBaseURL: (baseURL: string) => Site; + updateImsOrgId: (imsOrgId: string) => Site; + setAudits: (audits: Audit[]) => Site; } export interface DataAccess { @@ -64,6 +67,12 @@ export interface DataAccess { baseUrl: string, auditType: string, ) => Promise; + addSite: ( + siteData: object, + ) => Promise; + updateSite: ( + site: Site, + ) => Promise; } export function createDataAccess( diff --git a/packages/spacecat-shared-data-access/src/models/audit.js b/packages/spacecat-shared-data-access/src/models/audit.js index 76d7c37f5..2d5668df9 100644 --- a/packages/spacecat-shared-data-access/src/models/audit.js +++ b/packages/spacecat-shared-data-access/src/models/audit.js @@ -13,8 +13,15 @@ import { hasText, isIsoDate, isObject } from '@adobe/spacecat-shared-utils'; import { Base } from './base.js'; +export const AUDIT_TYPE_LHS = 'lhs'; + const EXPIRES_IN_DAYS = 30; +/** + * Creates a new Audit. + * @param {object } data - audit data + * @returns {Readonly} audit - new audit + */ const Audit = (data = {}) => { const self = Base(data); @@ -24,10 +31,33 @@ const Audit = (data = {}) => { self.getAuditType = () => self.state.auditType.toLowerCase(); self.getExpiresAt = () => self.state.expiresAt; self.getFullAuditRef = () => self.state.fullAuditRef; + self.getScores = () => { + const auditResult = self.getAuditResult(); + + if (self.getAuditType() === AUDIT_TYPE_LHS) { + const { + performance, seo, accessibility, bestPractices, + } = auditResult; + return { + performance, + seo, + accessibility, + bestPractices, + }; + } + + return auditResult; + }; return Object.freeze(self); }; +/** + * Creates a new Audit. + * + * @param {object } data - audit data + * @returns {Readonly} audit - new audit + */ export const createAudit = (data) => { const newState = { ...data }; diff --git a/packages/spacecat-shared-data-access/src/models/base.js b/packages/spacecat-shared-data-access/src/models/base.js index 797fa26fa..e75dd979f 100644 --- a/packages/spacecat-shared-data-access/src/models/base.js +++ b/packages/spacecat-shared-data-access/src/models/base.js @@ -13,6 +13,12 @@ import { v4 as uuidv4 } from 'uuid'; import { isString } from '@adobe/spacecat-shared-utils'; +/** + * Base model. + * + * @param {object} data data + * @returns {Base} base model + */ export const Base = (data = {}) => { const self = { state: { ...data } }; diff --git a/packages/spacecat-shared-data-access/src/models/site.js b/packages/spacecat-shared-data-access/src/models/site.js index b08587d05..61d5cb8b0 100644 --- a/packages/spacecat-shared-data-access/src/models/site.js +++ b/packages/spacecat-shared-data-access/src/models/site.js @@ -10,12 +10,19 @@ * governing permissions and limitations under the License. */ -import { hasText, isValidUrl } from '@adobe/spacecat-shared-utils'; +import { hasText, isObject, isValidUrl } from '@adobe/spacecat-shared-utils'; import { Base } from './base.js'; +/** + * Creates a new Site. + * + * @param {object} data - site data + * @returns {Readonly} site - new site + */ const Site = (data = {}) => { const self = Base(data); + self.getAudits = () => self.state.audits; self.getBaseURL = () => self.state.baseURL; self.getImsOrgId = () => self.state.imsOrgId; @@ -37,9 +44,20 @@ const Site = (data = {}) => { return self; }; + self.setAudits = (audits) => { + self.state.audits = audits; + return self; + }; + return Object.freeze(self); }; +/** + * Creates a new Site. + * + * @param {object} data - site data + * @returns {Readonly} site - new site + */ export const createSite = (data) => { const newState = { ...data }; @@ -47,5 +65,9 @@ export const createSite = (data) => { throw new Error('Base URL must be a valid URL'); } + if (!isObject(newState.audits)) { + newState.audits = {}; + } + return Site(newState); }; diff --git a/packages/spacecat-shared-data-access/src/sites/accessPatterns.js b/packages/spacecat-shared-data-access/src/sites/accessPatterns.js index ca862e493..7116b9d6e 100644 --- a/packages/spacecat-shared-data-access/src/sites/accessPatterns.js +++ b/packages/spacecat-shared-data-access/src/sites/accessPatterns.js @@ -10,38 +10,50 @@ * governing permissions and limitations under the License. */ +import { isObject } from '@adobe/spacecat-shared-utils'; + import { getAuditsForSite, getLatestAuditForSite, getLatestAudits, } from '../audits/accessPatterns.js'; +import { createSite } from '../models/site.js'; +import { SiteDto } from '../dto/site.js'; + +const INDEX_NAME_ALL_SITES = 'all_sites'; +const PK_ALL_SITES = 'ALL_SITES'; +const TABLE_NAME_SITES = 'sites'; + /** * Retrieves all sites. * * @param {DynamoDbClient} dynamoClient - The DynamoDB client. - * @returns {Promise} A promise that resolves to an array of all sites. + * @returns {Promise>} A promise that resolves to an array of all sites. */ -export const getSites = async (dynamoClient) => dynamoClient.query({ - TableName: 'sites', - IndexName: 'all_sites', // GSI name - KeyConditionExpression: 'GSI1PK = :gsi1pk', - ExpressionAttributeValues: { - ':gsi1pk': 'ALL_SITES', - }, -}); +export const getSites = async (dynamoClient) => { + const dynamoItems = await dynamoClient.query({ + TableName: TABLE_NAME_SITES, + IndexName: INDEX_NAME_ALL_SITES, // GSI name + KeyConditionExpression: 'GSI1PK = :gsi1pk', + ExpressionAttributeValues: { + ':gsi1pk': PK_ALL_SITES, + }, + }); + + return dynamoItems.map((dynamoItem) => SiteDto.fromDynamoItem(dynamoItem)); +}; /** * Retrieves a list of base URLs for all sites. * * @param {DynamoDbClient} dynamoClient - The DynamoDB client. - * @param {Logger} log - The logger. * @returns {Promise>} A promise that resolves to an array of base URLs for all sites. */ -export const getSitesToAudit = async (dynamoClient, log) => { - const sites = await getSites(dynamoClient, log); +export const getSitesToAudit = async (dynamoClient) => { + const sites = await getSites(dynamoClient); - return sites.map((item) => item.baseURL); + return sites.map((site) => site.getBaseURL()); }; /** @@ -66,12 +78,12 @@ export const getSitesWithLatestAudit = async ( getLatestAudits(dynamoClient, log, auditType, sortAuditsAscending), ]); - const sitesMap = new Map(sites.map((site) => [site.id, site])); + const sitesMap = new Map(sites.map((site) => [site.getId(), site])); return latestAudits.reduce((result, audit) => { - const site = sitesMap.get(audit.siteId); + const site = sitesMap.get(audit.getSiteId()); if (site) { - site.audits = [audit]; + site.setAudits([audit]); result.push(site); } return result; @@ -84,17 +96,25 @@ export const getSitesWithLatestAudit = async ( * @param {DynamoDbClient} dynamoClient - The DynamoDB client. * @param {Logger} log - The logger. * @param {string} baseUrl - The base URL of the site to retrieve. - * @returns {Promise} A promise that resolves to the site object if found, + * @returns {Promise} A promise that resolves to the site object if found, * otherwise null. */ export const getSiteByBaseURL = async ( dynamoClient, log, baseUrl, -) => dynamoClient.getItem('sites', { - GSI1PK: 'ALL_SITES', - baseUrl, -}); +) => { + const dynamoItem = await dynamoClient.getItem(TABLE_NAME_SITES, { + GSI1PK: PK_ALL_SITES, + baseUrl, + }); + + if (dynamoItem === null) { + return null; + } + + return SiteDto.fromDynamoItem(dynamoItem); +}; /** * Retrieves a site by its base URL, along with associated audit information. @@ -104,7 +124,7 @@ export const getSiteByBaseURL = async ( * @param {string} baseUrl - The base URL of the site to retrieve. * @param {string} auditType - The type of audits to retrieve for the site. * @param {boolean} [latestOnly=false] - Determines if only the latest audit should be retrieved. - * @returns {Promise} A promise that resolves to the site object with audit + * @returns {Promise} A promise that resolves to the site object with audit * data if found, otherwise null. */ export const getSiteByBaseURLWithAuditInfo = async ( @@ -116,24 +136,26 @@ export const getSiteByBaseURLWithAuditInfo = async ( ) => { const site = await getSiteByBaseURL(dynamoClient, log, baseUrl); - if (!site) { + if (!isObject(site)) { return null; } - site.audits = latestOnly + const audits = latestOnly ? [await getLatestAuditForSite( dynamoClient, log, - site.id, + site.getId(), auditType, )].filter((audit) => audit != null) : await getAuditsForSite( dynamoClient, log, - site.id, + site.getId(), auditType, ); + site.setAudits(audits); + return site; }; @@ -144,7 +166,7 @@ export const getSiteByBaseURLWithAuditInfo = async ( * @param {Logger} log - The logger. * @param {string} baseUrl - The base URL of the site to retrieve. * @param {string} auditType - The type of audits to retrieve for the site. - * @returns {Promise} A promise that resolves to the site object with all its audits. + * @returns {Promise} A promise that resolves to the site object with all its audits. */ export const getSiteByBaseURLWithAudits = async ( dynamoClient, @@ -160,7 +182,7 @@ export const getSiteByBaseURLWithAudits = async ( * @param {Logger} log - The logger. * @param {string} baseUrl - The base URL of the site to retrieve. * @param {string} auditType - The type of the latest audit to retrieve for the site. - * @returns {Promise} A promise that resolves to the site object with its latest audit. + * @returns {Promise} A promise that resolves to the site object with its latest audit. */ export const getSiteByBaseURLWithLatestAudit = async ( dynamoClient, @@ -168,3 +190,48 @@ export const getSiteByBaseURLWithLatestAudit = async ( baseUrl, auditType, ) => getSiteByBaseURLWithAuditInfo(dynamoClient, log, baseUrl, auditType, true); + +/** + * Adds a site. + * + * @param {DynamoDbClient} dynamoClient - The DynamoDB client. + * @param {Logger} log - The logger. + * @param {object} siteData - The site data. + * @returns {Promise>} + */ +export const addSite = async (dynamoClient, log, siteData) => { + const site = createSite(siteData); + const existingSite = await getSiteByBaseURL( + dynamoClient, + log, + site.getBaseURL(), + ); + + if (isObject(existingSite)) { + throw new Error('Site already exists'); + } + + await dynamoClient.putItem(TABLE_NAME_SITES, SiteDto.toDynamoItem(site)); + + return site; +}; + +/** + * Updates a site. + * + * @param {DynamoDbClient} dynamoClient - The DynamoDB client. + * @param {Logger} log - The logger. + * @param {Site} site - The site. + * @returns {Promise} - The updated site. + */ +export const updateSite = async (dynamoClient, log, site) => { + const existingSite = await getSiteByBaseURL(dynamoClient, log, site.getBaseURL()); + + if (!isObject(existingSite)) { + throw new Error('Site not found'); + } + + await dynamoClient.putItem(TABLE_NAME_SITES, SiteDto.toDynamoItem(site)); + + return site; +}; diff --git a/packages/spacecat-shared-data-access/src/sites/index.js b/packages/spacecat-shared-data-access/src/sites/index.js index e9ac693bd..1009eccbe 100644 --- a/packages/spacecat-shared-data-access/src/sites/index.js +++ b/packages/spacecat-shared-data-access/src/sites/index.js @@ -11,20 +11,23 @@ */ import { + addSite, getSiteByBaseURL, getSiteByBaseURLWithAuditInfo, getSiteByBaseURLWithAudits, - getSiteByBaseURLWithLatestAudit, getSites, getSitesToAudit, getSitesWithLatestAudit, + getSiteByBaseURLWithLatestAudit, + getSites, + getSitesToAudit, + getSitesWithLatestAudit, + updateSite, } from './accessPatterns.js'; export const siteFunctions = (dynamoClient, log) => ({ getSites: () => getSites( dynamoClient, - log, ), getSitesToAudit: () => getSitesToAudit( dynamoClient, - log, ), getSitesWithLatestAudit: (auditType, sortAuditsAscending) => getSitesWithLatestAudit( dynamoClient, @@ -56,4 +59,6 @@ export const siteFunctions = (dynamoClient, log) => ({ baseUrl, auditType, ), + addSite: (siteData) => addSite(dynamoClient, log, siteData), + updateSite: (site) => updateSite(dynamoClient, log, site), }); diff --git a/packages/spacecat-shared-data-access/test/audits/index.test.js b/packages/spacecat-shared-data-access/test/audits/index.test.js index 9e8a63db3..8c689596e 100644 --- a/packages/spacecat-shared-data-access/test/audits/index.test.js +++ b/packages/spacecat-shared-data-access/test/audits/index.test.js @@ -57,23 +57,98 @@ describe('Audit Index Tests', () => { it('calls getAuditsForSite and return an array', async () => { const result = await exportedFunctions.getAuditsForSite('siteId', 'auditType'); expect(result).to.be.an('array'); - // eslint-disable-next-line no-unused-expressions expect(mockDynamoClient.query.called).to.be.true; }); it('calls getLatestAudits and return an array', async () => { const result = await exportedFunctions.getLatestAudits('auditType', true); expect(result).to.be.an('array'); - // eslint-disable-next-line no-unused-expressions expect(mockDynamoClient.query.called).to.be.true; }); it('calls getLatestAuditForSite and return an array', async () => { const result = await exportedFunctions.getLatestAuditForSite('siteId', 'auditType'); - // eslint-disable-next-line no-unused-expressions expect(result).to.be.null; - // eslint-disable-next-line no-unused-expressions expect(mockDynamoClient.query.called).to.be.true; }); }); + + describe('getAuditForSite Tests', () => { + let mockDynamoClient; + let mockLog; + let exportedFunctions; + + beforeEach(() => { + mockDynamoClient = { + query: sinon.stub().returns(Promise.resolve([])), + }; + mockLog = { log: sinon.stub() }; + exportedFunctions = auditFunctions(mockDynamoClient, mockLog); + }); + + it('successfully retrieves an audit for a site', async () => { + const mockAuditData = [{ + siteId: 'siteId', + auditType: 'type1', + auditedAt: new Date().toISOString(), + auditResult: { score: 1 }, + fullAuditRef: 'https://someurl.com', + }]; + mockDynamoClient.query.returns(Promise.resolve(mockAuditData)); + + const result = await exportedFunctions.getAuditForSite('siteId', 'auditType', 'auditedAt'); + expect(result).to.not.be.null; + expect(result.getScores()).to.be.an('object'); + expect(mockDynamoClient.query.calledOnce).to.be.true; + }); + + it('returns null if no audit is found for a site', async () => { + mockDynamoClient.query.returns(Promise.resolve([])); + + const result = await exportedFunctions.getAuditForSite('siteId', 'auditType', 'auditedAt'); + expect(result).to.be.null; + expect(mockDynamoClient.query.calledOnce).to.be.true; + }); + }); + + describe('addAudit Tests', () => { + let mockDynamoClient; + let mockLog; + let exportedFunctions; + + const auditData = { + siteId: 'siteId', + auditType: 'lhs', + auditedAt: new Date().toISOString(), + auditResult: { score: 1 }, + fullAuditRef: 'https://someurl.com', + }; + + beforeEach(() => { + mockDynamoClient = { + query: sinon.stub().returns(Promise.resolve([])), + putItem: sinon.stub().returns(Promise.resolve()), + }; + mockLog = { log: sinon.stub() }; + exportedFunctions = auditFunctions(mockDynamoClient, mockLog); + }); + + it('successfully adds a new audit', async () => { + const result = await exportedFunctions.addAudit(auditData); + // Once for 'audits' and once for 'latest_audits' + expect(mockDynamoClient.putItem.calledTwice).to.be.true; + expect(result.getSiteId()).to.equal(auditData.siteId); + expect(result.getAuditType()).to.equal(auditData.auditType); + expect(result.getAuditedAt()).to.equal(auditData.auditedAt); + expect(result.getAuditResult()).to.deep.equal(auditData.auditResult); + expect(result.getFullAuditRef()).to.equal(auditData.fullAuditRef); + expect(result.getScores()).to.be.an('object'); + }); + + it('throws an error if audit already exists', async () => { + mockDynamoClient.query.returns(Promise.resolve([auditData])); + + await expect(exportedFunctions.addAudit(auditData)).to.be.rejectedWith('Audit already exists'); + }); + }); }); diff --git a/packages/spacecat-shared-data-access/test/index.test.js b/packages/spacecat-shared-data-access/test/index.test.js index 656b22525..d6c933f4a 100644 --- a/packages/spacecat-shared-data-access/test/index.test.js +++ b/packages/spacecat-shared-data-access/test/index.test.js @@ -17,11 +17,15 @@ import { createDataAccess } from '../src/index.js'; describe('Data Access Object Tests', () => { const auditFunctions = [ + 'addAudit', + 'getAuditForSite', 'getAuditsForSite', 'getLatestAudits', 'getLatestAuditForSite', ]; const siteFunctions = [ + 'addSite', + 'updateSite', 'getSites', 'getSitesToAudit', 'getSitesWithLatestAudit', diff --git a/packages/spacecat-shared-data-access/test/models/base.test.js b/packages/spacecat-shared-data-access/test/models/base.test.js index e9264aad9..0b2759434 100644 --- a/packages/spacecat-shared-data-access/test/models/base.test.js +++ b/packages/spacecat-shared-data-access/test/models/base.test.js @@ -38,7 +38,6 @@ describe('Base Entity Tests', () => { it('should return undefined for createdAt if not provided', () => { const baseEntity = Base(); - // eslint-disable-next-line no-unused-expressions expect(baseEntity.getCreatedAt()).to.be.undefined; }); @@ -50,7 +49,6 @@ describe('Base Entity Tests', () => { it('should return undefined for updatedAt if not provided', () => { const baseEntity = Base(); - // eslint-disable-next-line no-unused-expressions expect(baseEntity.getUpdatedAt()).to.be.undefined; }); }); diff --git a/packages/spacecat-shared-data-access/test/models/site.test.js b/packages/spacecat-shared-data-access/test/models/site.test.js index 899c8fadc..a195af4ff 100644 --- a/packages/spacecat-shared-data-access/test/models/site.test.js +++ b/packages/spacecat-shared-data-access/test/models/site.test.js @@ -38,7 +38,6 @@ describe('Site Module Tests', () => { let site; beforeEach(() => { - // Reset site object before each test site = createSite(validData); }); diff --git a/packages/spacecat-shared-data-access/test/sites/index.test.js b/packages/spacecat-shared-data-access/test/sites/index.test.js index a3874983b..0774d2516 100644 --- a/packages/spacecat-shared-data-access/test/sites/index.test.js +++ b/packages/spacecat-shared-data-access/test/sites/index.test.js @@ -12,10 +12,16 @@ /* eslint-env mocha */ -import { expect } from 'chai'; +import chai from 'chai'; +import chaiAsPromised from 'chai-as-promised'; import sinon from 'sinon'; import { siteFunctions } from '../../src/sites/index.js'; +import { createSite } from '../../src/models/site.js'; + +chai.use(chaiAsPromised); + +const { expect } = chai; describe('Site Index Tests', () => { describe('Site Functions Export Tests', () => { @@ -78,48 +84,46 @@ describe('Site Index Tests', () => { it('calls getSites and returns an array', async () => { const result = await exportedFunctions.getSites(); expect(result).to.be.an('array'); - // eslint-disable-next-line no-unused-expressions expect(mockDynamoClient.query.called).to.be.true; }); it('calls getSitesToAudit and returns an array', async () => { const result = await exportedFunctions.getSitesToAudit(); expect(result).to.be.an('array'); - // eslint-disable-next-line no-unused-expressions expect(mockDynamoClient.query.called).to.be.true; }); it('calls getSitesWithLatestAudit and returns an array', async () => { const result = await exportedFunctions.getSitesWithLatestAudit(); expect(result).to.be.an('array'); - // eslint-disable-next-line no-unused-expressions expect(mockDynamoClient.query.called).to.be.true; }); it('calls getSitesWithLatestAudit and handles latestAudits', async () => { const mockSiteData = [{ id: 'site1', - baseUrl: 'https://example.com', + baseURL: 'https://example.com', }]; const mockAuditData = [{ - id: 'audit1', siteId: 'site1', auditType: 'type1', + auditedAt: new Date().toISOString(), + auditResult: {}, + fullAuditRef: 'https://example.com', }]; mockDynamoClient.query.onFirstCall().resolves(mockSiteData); mockDynamoClient.query.onSecondCall().resolves(mockAuditData); const result = await exportedFunctions.getSitesWithLatestAudit('auditType'); - // eslint-disable-next-line no-unused-expressions expect(result).to.be.an('array').that.has.lengthOf(1); }); it('calls getSitesWithLatestAudit and handles empty latestAudits', async () => { const mockSiteData = [{ id: 'site1', - baseUrl: 'https://example.com', + baseURL: 'https://example.com', }]; const mockAuditData = []; @@ -128,23 +132,18 @@ describe('Site Index Tests', () => { mockDynamoClient.query.onSecondCall().resolves(mockAuditData); const result = await exportedFunctions.getSitesWithLatestAudit('auditType'); - // eslint-disable-next-line no-unused-expressions expect(result).to.be.an('array').that.is.empty; }); it('calls getSiteByBaseURL and returns an array/object', async () => { const result = await exportedFunctions.getSiteByBaseURL(); - // eslint-disable-next-line no-unused-expressions expect(result).to.be.null; - // eslint-disable-next-line no-unused-expressions expect(mockDynamoClient.getItem.called).to.be.true; }); it('calls getSiteByBaseURLWithAuditInfo and returns an array/object', async () => { const result = await exportedFunctions.getSiteByBaseURLWithAuditInfo(); - // eslint-disable-next-line no-unused-expressions expect(result).to.be.null; - // eslint-disable-next-line no-unused-expressions expect(mockDynamoClient.getItem.called).to.be.true; }); @@ -152,63 +151,151 @@ describe('Site Index Tests', () => { mockDynamoClient.query.resolves(undefined); const result = await exportedFunctions.getSiteByBaseURLWithAuditInfo('baseUrl', 'auditType'); - // eslint-disable-next-line no-unused-expressions expect(result).to.be.null; }); it('calls getSiteByBaseURLWithAuditInfo and assigns latest audit when latestOnly is true', async () => { const mockSiteData = { id: 'site1', - baseUrl: 'https://example.com', + baseURL: 'https://example.com', }; const mockLatestAuditData = [{ - id: 'audit1', siteId: 'site1', auditType: 'type1', + auditedAt: new Date().toISOString(), + auditResult: {}, + fullAuditRef: 'https://example.com', }]; mockDynamoClient.getItem.onFirstCall().resolves(mockSiteData); mockDynamoClient.query.onFirstCall().resolves(mockLatestAuditData); const result = await exportedFunctions.getSiteByBaseURLWithAuditInfo('https://example.com', 'type1', true); - expect(result).to.have.property('audits').that.is.an('array').with.lengthOf(1); - expect(result.audits[0]).to.deep.equal(mockLatestAuditData[0]); + const audits = result.getAudits(); + expect(audits).to.be.an('array').with.lengthOf(1); + + const audit = audits[0]; + expect(audit.getId()).to.be.a('string').that.is.not.empty; + expect(audit.getSiteId()).to.equal(mockLatestAuditData[0].siteId); + expect(audit.getAuditType()).to.equal(mockLatestAuditData[0].auditType); + expect(audit.getAuditedAt()).to.equal(mockLatestAuditData[0].auditedAt); + expect(audit.getAuditResult()).to.deep.equal(mockLatestAuditData[0].auditResult); + expect(audit.getFullAuditRef()).to.equal(mockLatestAuditData[0].fullAuditRef); }); it('calls getSiteByBaseURLWithAuditInfo and assigns all audits when latestOnly is false', async () => { const mockSiteData = { id: 'site1', - baseUrl: 'https://example.com', + baseURL: 'https://example.com', }; const mockLatestAuditData = [{ - id: 'audit1', siteId: 'site1', auditType: 'type1', + auditedAt: new Date().toISOString(), + auditResult: {}, + fullAuditRef: 'https://example.com', + }, + { + siteId: 'site1', + auditType: 'type2', + auditedAt: new Date().toISOString(), + auditResult: {}, + fullAuditRef: 'https://example2.com', }]; mockDynamoClient.getItem.onFirstCall().resolves(mockSiteData); mockDynamoClient.query.onFirstCall().resolves(mockLatestAuditData); const result = await exportedFunctions.getSiteByBaseURLWithAuditInfo('baseUrl', 'auditType', false); - expect(result).to.have.property('audits').that.is.an('array'); + const audits = result.getAudits(); + expect(audits).to.be.an('array').with.lengthOf(2); + + for (let i = 0; i < mockLatestAuditData.length; i += 1) { + const mockAudit = mockLatestAuditData[i]; + const audit = audits[i]; + + expect(audit.getId()).to.be.a('string').that.is.not.empty; + expect(audit.getSiteId()).to.equal(mockAudit.siteId); + expect(audit.getAuditType()).to.equal(mockAudit.auditType); + expect(audit.getAuditedAt()).to.equal(mockAudit.auditedAt); + expect(audit.getAuditResult()).to.deep.equal(mockAudit.auditResult); + expect(audit.getFullAuditRef()).to.equal(mockAudit.fullAuditRef); + } }); it('calls getSiteByBaseURLWithAudits and returns an array/object', async () => { const result = await exportedFunctions.getSiteByBaseURLWithAudits(); - // eslint-disable-next-line no-unused-expressions expect(result).to.be.null; - // eslint-disable-next-line no-unused-expressions expect(mockDynamoClient.getItem.called).to.be.true; }); it('calls getSiteByBaseURLWithLatestAudit and returns an array/object', async () => { const result = await exportedFunctions.getSiteByBaseURLWithLatestAudit(); - // eslint-disable-next-line no-unused-expressions expect(result).to.be.null; - // eslint-disable-next-line no-unused-expressions expect(mockDynamoClient.getItem.called).to.be.true; }); + + describe('addSite Tests', () => { + beforeEach(() => { + mockDynamoClient = { + getItem: sinon.stub().returns(Promise.resolve(null)), + putItem: sinon.stub().returns(Promise.resolve()), + }; + mockLog = { log: sinon.stub() }; + exportedFunctions = siteFunctions(mockDynamoClient, mockLog); + }); + + it('adds a new site successfully', async () => { + const siteData = { baseURL: 'https://newsite.com' }; + const result = await exportedFunctions.addSite(siteData); + expect(mockDynamoClient.putItem.calledOnce).to.be.true; + expect(result.getBaseURL()).to.equal(siteData.baseURL); + expect(result.getId()).to.be.a('string'); + expect(result.getAudits()).to.be.an('object').that.is.empty; + }); + + it('throws an error if site already exists', async () => { + const siteData = { baseURL: 'https://existingsite.com' }; + mockDynamoClient.getItem.returns(Promise.resolve(siteData)); + + await expect(exportedFunctions.addSite(siteData)).to.be.rejectedWith('Site already exists'); + }); + }); + }); + + describe('updateSite Tests', () => { + let mockDynamoClient; + let mockLog; + let exportedFunctions; + + beforeEach(() => { + mockDynamoClient = { + getItem: sinon.stub().returns(Promise.resolve(null)), + putItem: sinon.stub().returns(Promise.resolve()), + }; + mockLog = { log: sinon.stub() }; + exportedFunctions = siteFunctions(mockDynamoClient, mockLog); + }); + + it('updates an existing site successfully', async () => { + const siteData = { baseURL: 'https://existingsite.com' }; + mockDynamoClient.getItem.returns(Promise.resolve(siteData)); + + const site = await exportedFunctions.getSiteByBaseURL(siteData.baseURL); + site.updateBaseURL('https://newsite.com'); + site.updateImsOrgId('newOrg123'); + + const result = await exportedFunctions.updateSite(site); + expect(mockDynamoClient.putItem.calledOnce).to.be.true; + expect(result.getBaseURL()).to.equal(site.getBaseURL()); + expect(result.getImsOrgId()).to.equal(site.getImsOrgId()); + }); + + it('throws an error if site does not exist', async () => { + const site = createSite({ baseURL: 'https://nonexistingsite.com' }); + await expect(exportedFunctions.updateSite(site)).to.be.rejectedWith('Site not found'); + }); }); }); diff --git a/packages/spacecat-shared-dynamo/src/index.d.ts b/packages/spacecat-shared-dynamo/src/index.d.ts index 8599d9c16..21bb25c46 100644 --- a/packages/spacecat-shared-dynamo/src/index.d.ts +++ b/packages/spacecat-shared-dynamo/src/index.d.ts @@ -18,16 +18,11 @@ export declare interface Logger { info(message: string, ...args: unknown[]): void; } -export declare interface DynamoDbKey { - partitionKey: string; - sortKey?: string; -} - export declare interface DynamoDbClient { query(originalParams: QueryCommandInput): Promise; - getItem(tableName: string, key: DynamoDbKey): Promise; + getItem(tableName: string, key: object): Promise; putItem(tableName: string, item: object): Promise<{ message: string }>; - removeItem(tableName: string, key: DynamoDbKey): Promise<{ message: string }>; + removeItem(tableName: string, key: object): Promise<{ message: string }>; } export function createClient( diff --git a/packages/spacecat-shared-dynamo/src/modules/getItem.js b/packages/spacecat-shared-dynamo/src/modules/getItem.js index 5a4e24e8a..ba0d32232 100644 --- a/packages/spacecat-shared-dynamo/src/modules/getItem.js +++ b/packages/spacecat-shared-dynamo/src/modules/getItem.js @@ -19,7 +19,7 @@ import { guardKey, guardTableName } from '../utils/guards.js'; * * @param {DynamoDBDocumentClient} docClient - The AWS SDK DynamoDB Document client instance. * @param {string} tableName - The name of the DynamoDB table. - * @param {DynamoDbKey} key - The key object containing partitionKey and optionally sortKey. + * @param {object} key - The key object containing partitionKey and optionally sortKey. * @param {Logger} log - The logging object, defaults to console. * @returns {Promise} A promise that resolves to the retrieved item. * @throws {Error} Throws an error if the DynamoDB get operation fails or input validation fails. diff --git a/packages/spacecat-shared-dynamo/src/modules/removeItem.js b/packages/spacecat-shared-dynamo/src/modules/removeItem.js index 2978278ca..3ab723e9b 100644 --- a/packages/spacecat-shared-dynamo/src/modules/removeItem.js +++ b/packages/spacecat-shared-dynamo/src/modules/removeItem.js @@ -19,7 +19,7 @@ import { guardKey, guardTableName } from '../utils/guards.js'; * * @param {DynamoDBDocumentClient} docClient - The AWS SDK DynamoDB Document client instance. * @param {string} tableName - The name of the DynamoDB table. - * @param {DynamoDbKey} key - The key object containing partitionKey and optionally sortKey. + * @param {object} key - The key object containing partitionKey and optionally sortKey. * @param {Logger} log - The logging object, defaults to console. * @returns {Promise} A promise that resolves to a message indicating successful removal. * @throws {Error} Throws an error if the DynamoDB delete operation fails or input validation fails. From 9954c80dea332ca119cea1f78ba38017b48e4d94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominique=20J=C3=A4ggi?= Date: Thu, 30 Nov 2023 11:57:59 +0100 Subject: [PATCH 05/28] fix: use correct doc client --- packages/spacecat-shared-dynamo/src/index.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/spacecat-shared-dynamo/src/index.js b/packages/spacecat-shared-dynamo/src/index.js index 6b09ed72e..51301776a 100644 --- a/packages/spacecat-shared-dynamo/src/index.js +++ b/packages/spacecat-shared-dynamo/src/index.js @@ -11,7 +11,7 @@ */ import { DynamoDB } from '@aws-sdk/client-dynamodb'; -import { DynamoDBDocumentClient } from '@aws-sdk/lib-dynamodb'; +import { DynamoDBDocument } from '@aws-sdk/lib-dynamodb'; import query from './modules/query.js'; import getItem from './modules/getItem.js'; @@ -23,13 +23,13 @@ import removeItem from './modules/removeItem.js'; * * @param {Object} log - The logging object, defaults to console. * @param {DynamoDB} dbClient - The AWS SDK DynamoDB client instance. - * @param {DynamoDBDocumentClient} docClient - The AWS SDK DynamoDB Document client instance. + * @param {DynamoDBDocument} docClient - The AWS SDK DynamoDB Document client instance. * @returns {Object} A client object with methods to interact with DynamoDB. */ const createClient = ( log = console, dbClient = new DynamoDB(), - docClient = DynamoDBDocumentClient.from(dbClient), + docClient = DynamoDBDocument.from(dbClient), ) => ({ query: (params) => query(docClient, params, log), getItem: (tableName, key) => getItem(docClient, tableName, key, log), From 41f4782c2f0a6ed6b930a47c451fd004e9d54ea5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominique=20J=C3=A4ggi?= Date: Thu, 30 Nov 2023 14:11:54 +0100 Subject: [PATCH 06/28] fix: allow use of index for getItem --- packages/spacecat-shared-dynamo/src/index.d.ts | 2 +- packages/spacecat-shared-dynamo/src/index.js | 2 +- .../src/modules/getItem.js | 17 ++++++++++++++++- .../test/modules/getItem.test.js | 14 ++++++++++---- 4 files changed, 28 insertions(+), 7 deletions(-) diff --git a/packages/spacecat-shared-dynamo/src/index.d.ts b/packages/spacecat-shared-dynamo/src/index.d.ts index 21bb25c46..f49671ab6 100644 --- a/packages/spacecat-shared-dynamo/src/index.d.ts +++ b/packages/spacecat-shared-dynamo/src/index.d.ts @@ -20,7 +20,7 @@ export declare interface Logger { export declare interface DynamoDbClient { query(originalParams: QueryCommandInput): Promise; - getItem(tableName: string, key: object): Promise; + getItem(tableName: string, indexName: string, key: object): Promise; putItem(tableName: string, item: object): Promise<{ message: string }>; removeItem(tableName: string, key: object): Promise<{ message: string }>; } diff --git a/packages/spacecat-shared-dynamo/src/index.js b/packages/spacecat-shared-dynamo/src/index.js index 51301776a..e42d60ba8 100644 --- a/packages/spacecat-shared-dynamo/src/index.js +++ b/packages/spacecat-shared-dynamo/src/index.js @@ -32,7 +32,7 @@ const createClient = ( docClient = DynamoDBDocument.from(dbClient), ) => ({ query: (params) => query(docClient, params, log), - getItem: (tableName, key) => getItem(docClient, tableName, key, log), + getItem: (tableName, indexName, key) => getItem(docClient, tableName, indexName, key, log), putItem: (tableName, item) => putItem(docClient, tableName, item, log), removeItem: (tableName, key) => removeItem(docClient, tableName, key, log), }); diff --git a/packages/spacecat-shared-dynamo/src/modules/getItem.js b/packages/spacecat-shared-dynamo/src/modules/getItem.js index ba0d32232..164307616 100644 --- a/packages/spacecat-shared-dynamo/src/modules/getItem.js +++ b/packages/spacecat-shared-dynamo/src/modules/getItem.js @@ -12,6 +12,8 @@ import { performance } from 'perf_hooks'; +import { hasText } from '@adobe/spacecat-shared-utils'; + import { guardKey, guardTableName } from '../utils/guards.js'; /** @@ -19,18 +21,31 @@ import { guardKey, guardTableName } from '../utils/guards.js'; * * @param {DynamoDBDocumentClient} docClient - The AWS SDK DynamoDB Document client instance. * @param {string} tableName - The name of the DynamoDB table. + * @param {string} [indexName] - Optional. The name of the DynamoDB index. * @param {object} key - The key object containing partitionKey and optionally sortKey. * @param {Logger} log - The logging object, defaults to console. * @returns {Promise} A promise that resolves to the retrieved item. * @throws {Error} Throws an error if the DynamoDB get operation fails or input validation fails. */ -async function getItem(docClient, tableName, key, log = console) { +async function getItem( + docClient, + tableName, + indexName, + key, + log = console, +) { guardTableName(tableName); guardKey(key); + const indexProperties = {}; + if (hasText(indexName)) { + indexProperties.IndexName = indexName; + } + const params = { TableName: tableName, Key: key, + ...indexProperties, }; try { diff --git a/packages/spacecat-shared-dynamo/test/modules/getItem.test.js b/packages/spacecat-shared-dynamo/test/modules/getItem.test.js index 205c0101b..677b19c0e 100644 --- a/packages/spacecat-shared-dynamo/test/modules/getItem.test.js +++ b/packages/spacecat-shared-dynamo/test/modules/getItem.test.js @@ -29,13 +29,19 @@ describe('getItem', () => { it('gets an item from the database', async () => { const key = { partitionKey: 'testPartitionKey' }; - const result = await dynamoDbClient.getItem('TestTable', key); + const result = await dynamoDbClient.getItem('TestTable', null, key); expect(result).to.be.an('object'); }); it('gets an item from the database with sort key', async () => { const key = { partitionKey: 'testPartitionKey', sortKey: 'testSortKey' }; - const result = await dynamoDbClient.getItem('TestTable', key); + const result = await dynamoDbClient.getItem('TestTable', null, key); + expect(result).to.be.an('object'); + }); + + it('gets an item from the database with index', async () => { + const key = { partitionKey: 'testPartitionKey', sortKey: 'testSortKey' }; + const result = await dynamoDbClient.getItem('TestTable', 'test-index', key); expect(result).to.be.an('object'); }); @@ -51,7 +57,7 @@ describe('getItem', () => { it('throws an error for getItem with invalid key', async () => { try { - await dynamoDbClient.getItem('TestTable', null); + await dynamoDbClient.getItem('TestTable', null, null); expect.fail('getItem did not throw with invalid key'); } catch (error) { expect(error.message).to.equal('Key must be a non-empty object.'); @@ -64,7 +70,7 @@ describe('getItem', () => { }; try { - await dynamoDbClient.getItem('TestTable', { partitionKey: 'testPartitionKey' }); + await dynamoDbClient.getItem('TestTable', null, { partitionKey: 'testPartitionKey' }); expect.fail('getItem did not throw as expected'); } catch (error) { expect(error.message).to.equal('Get failed'); From 6b94f3823d50d3cbc1776dd26decd497af392a28 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominique=20J=C3=A4ggi?= Date: Thu, 30 Nov 2023 14:52:56 +0100 Subject: [PATCH 07/28] feat: add data access --- package-lock.json | 225 +++++++++++++++++- .../spacecat-shared-data-access/package.json | 8 +- .../src/models/audit.js | 4 +- .../src/models/site.js | 2 +- .../src/sites/accessPatterns.js | 20 +- .../test-it/auditUtils.js | 58 +++++ .../spacecat-shared-data-access/test-it/db.js | 22 ++ .../test-it/db.test.js | 112 +++++++++ .../test-it/generateSampleData.js | 168 +++++++++++++ .../test-it/tableOperations.js | 161 +++++++++++++ .../test-it/util.js | 22 ++ .../test/models/audit.test.js | 1 - .../test/sites/index.test.js | 36 +-- 13 files changed, 802 insertions(+), 37 deletions(-) create mode 100644 packages/spacecat-shared-data-access/test-it/auditUtils.js create mode 100644 packages/spacecat-shared-data-access/test-it/db.js create mode 100644 packages/spacecat-shared-data-access/test-it/db.test.js create mode 100644 packages/spacecat-shared-data-access/test-it/generateSampleData.js create mode 100644 packages/spacecat-shared-data-access/test-it/tableOperations.js create mode 100644 packages/spacecat-shared-data-access/test-it/util.js diff --git a/package-lock.json b/package-lock.json index eb9658331..b5cb53622 100644 --- a/package-lock.json +++ b/package-lock.json @@ -94,6 +94,10 @@ "aws4": "1.12.0" } }, + "node_modules/@adobe/spacecat-shared-data-access": { + "resolved": "packages/spacecat-shared-data-access", + "link": true + }, "node_modules/@adobe/spacecat-shared-dynamo": { "resolved": "packages/spacecat-shared-dynamo", "link": true @@ -1487,6 +1491,50 @@ "semantic-release": ">=18.0.0-beta.1" } }, + "node_modules/@sinonjs/commons": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-3.0.0.tgz", + "integrity": "sha512-jXBtWAF4vmdNmZgD5FoKsVLv3rPgDnLgPbU84LIJ3otV44vJlDRokVng5v8NFJdCf/da9legHcKaRuZs4L7faA==", + "dev": true, + "dependencies": { + "type-detect": "4.0.8" + } + }, + "node_modules/@sinonjs/fake-timers": { + "version": "11.2.2", + "resolved": "https://registry.npmjs.org/@sinonjs/fake-timers/-/fake-timers-11.2.2.tgz", + "integrity": "sha512-G2piCSxQ7oWOxwGSAyFHfPIsyeJGXYtc6mFbnFA+kRXkiEnTl8c/8jul2S329iFBnDI9HGoeWWAZvuvOkZccgw==", + "dev": true, + "dependencies": { + "@sinonjs/commons": "^3.0.0" + } + }, + "node_modules/@sinonjs/samsam": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/@sinonjs/samsam/-/samsam-8.0.0.tgz", + "integrity": "sha512-Bp8KUVlLp8ibJZrnvq2foVhP0IVX2CIprMJPK0vqGqgrDa0OHVKeZyBykqskkrdxV6yKBPmGasO8LVjAKR3Gew==", + "dev": true, + "dependencies": { + "@sinonjs/commons": "^2.0.0", + "lodash.get": "^4.4.2", + "type-detect": "^4.0.8" + } + }, + "node_modules/@sinonjs/samsam/node_modules/@sinonjs/commons": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-2.0.0.tgz", + "integrity": "sha512-uLa0j859mMrg2slwQYdO/AkrOfmH+X6LTVmNTS9CqexuE2IvVORIkSpJLqePAbEnKJ77aMmCwr1NUZ57120Xcg==", + "dev": true, + "dependencies": { + "type-detect": "4.0.8" + } + }, + "node_modules/@sinonjs/text-encoding": { + "version": "0.7.2", + "resolved": "https://registry.npmjs.org/@sinonjs/text-encoding/-/text-encoding-0.7.2.tgz", + "integrity": "sha512-sXXKG+uL9IrKqViTtao2Ws6dy0znu9sOaP1di/jKGW1M6VssO8vlpXCQcpZ+jisQ1tTFAC5Jo/EOzFbggBagFQ==", + "dev": true + }, "node_modules/@smithy/abort-controller": { "version": "2.0.14", "resolved": "https://registry.npmjs.org/@smithy/abort-controller/-/abort-controller-2.0.14.tgz", @@ -2808,6 +2856,18 @@ "node": ">=4" } }, + "node_modules/chai-as-promised": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/chai-as-promised/-/chai-as-promised-7.1.1.tgz", + "integrity": "sha512-azL6xMoi+uxu6z4rhWQ1jbdUhOMhis2PvscD/xjLqNMkv3BPPp2JyyuTHOrf9BOosGpNQ11v6BKv/g57RXbiaA==", + "dev": true, + "dependencies": { + "check-error": "^1.0.2" + }, + "peerDependencies": { + "chai": ">= 2.1.2 < 5" + } + }, "node_modules/chalk": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", @@ -3587,6 +3647,15 @@ "readable-stream": "^2.0.2" } }, + "node_modules/dynamo-db-local": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/dynamo-db-local/-/dynamo-db-local-6.1.0.tgz", + "integrity": "sha512-YdF22fGHJzWoHcO9X8soOgCF1Q7AmEf/0fSEovzXCMSOZJ/Mb8nXvyMj4s75aswLJfb8Ujyd64HdFYiofC9Mkg==", + "dev": true, + "engines": { + "node": ">=16.1.0" + } + }, "node_modules/eastasianwidth": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz", @@ -5584,6 +5653,12 @@ "node": "*" } }, + "node_modules/just-extend": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/just-extend/-/just-extend-4.2.1.tgz", + "integrity": "sha512-g3UB796vUFIY90VIv/WX3L2c8CS2MdWUww3CNrYmqza1Fg0DURc2K/O4YrnklBdQarSJ/y8JnJYDGc+1iumQjg==", + "dev": true + }, "node_modules/keyv": { "version": "4.5.4", "resolved": "https://registry.npmjs.org/keyv/-/keyv-4.5.4.tgz", @@ -5905,6 +5980,12 @@ "integrity": "sha512-TM9YBvyC84ZxE3rgfefxUWiQKLilstD6k7PTGt6wfbtXF8ixIJLOL3VYyV/z+ZiPLsVxAsKAFVwWlWeb2Y8Yyw==", "dev": true }, + "node_modules/lodash.get": { + "version": "4.4.2", + "resolved": "https://registry.npmjs.org/lodash.get/-/lodash.get-4.4.2.tgz", + "integrity": "sha512-z+Uw/vLuy6gQe8cfaFWD7p0wVv8fJl3mbzXh33RS+0oW2wvUqiRXiQ69gLWSLpgB5/6sU+r6BlQR0MBILadqTQ==", + "dev": true + }, "node_modules/lodash.ismatch": { "version": "4.4.0", "resolved": "https://registry.npmjs.org/lodash.ismatch/-/lodash.ismatch-4.4.0.tgz", @@ -6616,6 +6697,46 @@ "integrity": "sha512-EZSPZB70jiVsivaBLYDCyntd5eH8NTSMOn3rB+HxwdmKThGELLdYv8qVIMWvZEFy9w8ZZpW9h9OB32l1rGtj7g==", "dev": true }, + "node_modules/nise": { + "version": "5.1.5", + "resolved": "https://registry.npmjs.org/nise/-/nise-5.1.5.tgz", + "integrity": "sha512-VJuPIfUFaXNRzETTQEEItTOP8Y171ijr+JLq42wHes3DiryR8vT+1TXQW/Rx8JNUhyYYWyIvjXTU6dOhJcs9Nw==", + "dev": true, + "dependencies": { + "@sinonjs/commons": "^2.0.0", + "@sinonjs/fake-timers": "^10.0.2", + "@sinonjs/text-encoding": "^0.7.1", + "just-extend": "^4.0.2", + "path-to-regexp": "^1.7.0" + } + }, + "node_modules/nise/node_modules/@sinonjs/commons": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-2.0.0.tgz", + "integrity": "sha512-uLa0j859mMrg2slwQYdO/AkrOfmH+X6LTVmNTS9CqexuE2IvVORIkSpJLqePAbEnKJ77aMmCwr1NUZ57120Xcg==", + "dev": true, + "dependencies": { + "type-detect": "4.0.8" + } + }, + "node_modules/nise/node_modules/@sinonjs/fake-timers": { + "version": "10.3.0", + "resolved": "https://registry.npmjs.org/@sinonjs/fake-timers/-/fake-timers-10.3.0.tgz", + "integrity": "sha512-V4BG07kuYSUkTCSBHG8G8TNhM+F19jXFWnQtzj+we8DrkpSBCee9Z3Ms8yiGer/dlmhe35/Xdgyo3/0rQKg7YA==", + "dev": true, + "dependencies": { + "@sinonjs/commons": "^3.0.0" + } + }, + "node_modules/nise/node_modules/@sinonjs/fake-timers/node_modules/@sinonjs/commons": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-3.0.0.tgz", + "integrity": "sha512-jXBtWAF4vmdNmZgD5FoKsVLv3rPgDnLgPbU84LIJ3otV44vJlDRokVng5v8NFJdCf/da9legHcKaRuZs4L7faA==", + "dev": true, + "dependencies": { + "type-detect": "4.0.8" + } + }, "node_modules/nock": { "version": "13.3.8", "resolved": "https://registry.npmjs.org/nock/-/nock-13.3.8.tgz", @@ -9598,6 +9719,21 @@ "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==", "dev": true }, + "node_modules/path-to-regexp": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-1.8.0.tgz", + "integrity": "sha512-n43JRhlUKUAlibEJhPeir1ncUID16QnEjNpwzNdO3Lm4ywrBpBZ5oLD0I6br9evr1Y9JTqwRtAh7JLoOzAQdVA==", + "dev": true, + "dependencies": { + "isarray": "0.0.1" + } + }, + "node_modules/path-to-regexp/node_modules/isarray": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz", + "integrity": "sha512-D2S+3GLxWH+uhrNEcoh/fnmYeP8E8/zHl644d/jdA0g2uyXvy3sb0qxotE+ne0LtccHknQzWwZEzhak7oJ0COQ==", + "dev": true + }, "node_modules/path-type": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/path-type/-/path-type-4.0.0.tgz", @@ -10906,6 +11042,33 @@ "node": ">=4" } }, + "node_modules/sinon": { + "version": "17.0.1", + "resolved": "https://registry.npmjs.org/sinon/-/sinon-17.0.1.tgz", + "integrity": "sha512-wmwE19Lie0MLT+ZYNpDymasPHUKTaZHUH/pKEubRXIzySv9Atnlw+BUMGCzWgV7b7wO+Hw6f1TEOr0IUnmU8/g==", + "dev": true, + "dependencies": { + "@sinonjs/commons": "^3.0.0", + "@sinonjs/fake-timers": "^11.2.2", + "@sinonjs/samsam": "^8.0.0", + "diff": "^5.1.0", + "nise": "^5.1.5", + "supports-color": "^7.2.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/sinon" + } + }, + "node_modules/sinon/node_modules/diff": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/diff/-/diff-5.1.0.tgz", + "integrity": "sha512-D+mk+qE8VC/PAUrlAU34N+VfXev0ghe5ywmpqrawphmVZc1bEfn56uo9qpyGp1p4xpzOHkSW4ztBd6L7Xx4ACw==", + "dev": true, + "engines": { + "node": ">=0.3.1" + } + }, "node_modules/slash": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/slash/-/slash-3.0.0.tgz", @@ -12096,12 +12259,62 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "packages/spacecat-shared-data-access": { + "name": "@adobe/spacecat-shared-data-access", + "version": "1.1.0", + "license": "Apache-2.0", + "dependencies": { + "@adobe/spacecat-shared-dynamo": "1.1.4", + "@adobe/spacecat-shared-utils": "1.1.0", + "uuid": "9.0.1" + }, + "devDependencies": { + "@aws-sdk/client-dynamodb": "3.454.0", + "@aws-sdk/lib-dynamodb": "3.454.0", + "chai": "4.3.10", + "chai-as-promised": "7.1.1", + "dynamo-db-local": "6.1.0", + "sinon": "17.0.1" + } + }, + "packages/spacecat-shared-data-access/node_modules/@adobe/spacecat-shared-dynamo": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/@adobe/spacecat-shared-dynamo/-/spacecat-shared-dynamo-1.1.4.tgz", + "integrity": "sha512-geJZoXHOH3uZ5x3pFn05C1Vu5yrS2vsequYdxRLk03BjyynhLBuW0tT3vvTc9sTfFf1ZuEGhiT9hdWOQvJFh2g==", + "dependencies": { + "@adobe/spacecat-shared-utils": "1.0.1", + "@aws-sdk/client-dynamodb": "3.454.0", + "@aws-sdk/lib-dynamodb": "3.454.0" + } + }, + "packages/spacecat-shared-data-access/node_modules/@adobe/spacecat-shared-dynamo/node_modules/@adobe/spacecat-shared-utils": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@adobe/spacecat-shared-utils/-/spacecat-shared-utils-1.0.1.tgz", + "integrity": "sha512-6HiCywan5y9jpbFYkQX87Vg0q5dXoJf2m6MiDvNBEJdREdUxFwE+oYA6Fr86qqd32ECnCeYCFO13HcNRn1RxkQ==" + }, + "packages/spacecat-shared-data-access/node_modules/@adobe/spacecat-shared-utils": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@adobe/spacecat-shared-utils/-/spacecat-shared-utils-1.1.0.tgz", + "integrity": "sha512-yq6o47d6IuMApgvlGWdIfUOkqYNwcEm4IzO5WHnrTpWXPK4vvssfAfFtoeki7NQ+A4nFoc7/esqOfX42Q452nA==" + }, + "packages/spacecat-shared-data-access/node_modules/uuid": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", + "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==", + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "bin": { + "uuid": "dist/bin/uuid" + } + }, "packages/spacecat-shared-dynamo": { "name": "@adobe/spacecat-shared-dynamo", - "version": "1.1.2", + "version": "1.1.3", "license": "Apache-2.0", "dependencies": { - "@adobe/spacecat-shared-utils": "1.0.1", + "@adobe/spacecat-shared-utils": "1.1.0", "@aws-sdk/client-dynamodb": "3.454.0", "@aws-sdk/lib-dynamodb": "3.454.0" }, @@ -12110,9 +12323,9 @@ } }, "packages/spacecat-shared-dynamo/node_modules/@adobe/spacecat-shared-utils": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/@adobe/spacecat-shared-utils/-/spacecat-shared-utils-1.0.1.tgz", - "integrity": "sha512-6HiCywan5y9jpbFYkQX87Vg0q5dXoJf2m6MiDvNBEJdREdUxFwE+oYA6Fr86qqd32ECnCeYCFO13HcNRn1RxkQ==" + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@adobe/spacecat-shared-utils/-/spacecat-shared-utils-1.1.0.tgz", + "integrity": "sha512-yq6o47d6IuMApgvlGWdIfUOkqYNwcEm4IzO5WHnrTpWXPK4vvssfAfFtoeki7NQ+A4nFoc7/esqOfX42Q452nA==" }, "packages/spacecat-shared-example": { "name": "@adobe/spacecat-shared-example", @@ -12128,7 +12341,7 @@ }, "packages/spacecat-shared-utils": { "name": "@adobe/spacecat-shared-utils", - "version": "1.1.0", + "version": "1.2.0", "license": "Apache-2.0", "devDependencies": { "chai": "4.3.10" diff --git a/packages/spacecat-shared-data-access/package.json b/packages/spacecat-shared-data-access/package.json index 87c38fd66..90fea4b06 100644 --- a/packages/spacecat-shared-data-access/package.json +++ b/packages/spacecat-shared-data-access/package.json @@ -6,7 +6,8 @@ "main": "src/index.js", "types": "src/index.d.ts", "scripts": { - "test": "c8 mocha", + "test:it": "mocha test-it/", + "test": "c8 mocha test/", "lint": "eslint .", "clean": "rm -rf package-lock.json node_modules" }, @@ -29,13 +30,16 @@ "access": "public" }, "dependencies": { - "@adobe/spacecat-shared-dynamo": "1.1.2", + "@adobe/spacecat-shared-dynamo": "1.1.4", "@adobe/spacecat-shared-utils": "1.1.0", + "@aws-sdk/client-dynamodb": "3.454.0", + "@aws-sdk/lib-dynamodb": "3.454.0", "uuid": "9.0.1" }, "devDependencies": { "chai": "4.3.10", "chai-as-promised": "7.1.1", + "dynamo-db-local": "6.1.0", "sinon": "17.0.1" } } diff --git a/packages/spacecat-shared-data-access/src/models/audit.js b/packages/spacecat-shared-data-access/src/models/audit.js index 2d5668df9..6f603c1b5 100644 --- a/packages/spacecat-shared-data-access/src/models/audit.js +++ b/packages/spacecat-shared-data-access/src/models/audit.js @@ -36,13 +36,13 @@ const Audit = (data = {}) => { if (self.getAuditType() === AUDIT_TYPE_LHS) { const { - performance, seo, accessibility, bestPractices, + performance, seo, accessibility, 'best-practices': bestPractices, } = auditResult; return { performance, seo, accessibility, - bestPractices, + 'best-practices': bestPractices, }; } diff --git a/packages/spacecat-shared-data-access/src/models/site.js b/packages/spacecat-shared-data-access/src/models/site.js index 61d5cb8b0..10eee5201 100644 --- a/packages/spacecat-shared-data-access/src/models/site.js +++ b/packages/spacecat-shared-data-access/src/models/site.js @@ -66,7 +66,7 @@ export const createSite = (data) => { } if (!isObject(newState.audits)) { - newState.audits = {}; + newState.audits = []; } return Site(newState); diff --git a/packages/spacecat-shared-data-access/src/sites/accessPatterns.js b/packages/spacecat-shared-data-access/src/sites/accessPatterns.js index 7116b9d6e..7881d69de 100644 --- a/packages/spacecat-shared-data-access/src/sites/accessPatterns.js +++ b/packages/spacecat-shared-data-access/src/sites/accessPatterns.js @@ -95,25 +95,31 @@ export const getSitesWithLatestAudit = async ( * * @param {DynamoDbClient} dynamoClient - The DynamoDB client. * @param {Logger} log - The logger. - * @param {string} baseUrl - The base URL of the site to retrieve. + * @param {string} baseURL - The base URL of the site to retrieve. * @returns {Promise} A promise that resolves to the site object if found, * otherwise null. */ export const getSiteByBaseURL = async ( dynamoClient, log, - baseUrl, + baseURL, ) => { - const dynamoItem = await dynamoClient.getItem(TABLE_NAME_SITES, { - GSI1PK: PK_ALL_SITES, - baseUrl, + const dynamoItems = await dynamoClient.query({ + TableName: TABLE_NAME_SITES, + IndexName: INDEX_NAME_ALL_SITES, + KeyConditionExpression: 'GSI1PK = :gsi1pk AND baseURL = :baseURL', + ExpressionAttributeValues: { + ':gsi1pk': PK_ALL_SITES, + ':baseURL': baseURL, + }, + Limit: 1, }); - if (dynamoItem === null) { + if (dynamoItems.length === 0) { return null; } - return SiteDto.fromDynamoItem(dynamoItem); + return SiteDto.fromDynamoItem(dynamoItems[0]); }; /** diff --git a/packages/spacecat-shared-data-access/test-it/auditUtils.js b/packages/spacecat-shared-data-access/test-it/auditUtils.js new file mode 100644 index 000000000..9b0babd82 --- /dev/null +++ b/packages/spacecat-shared-data-access/test-it/auditUtils.js @@ -0,0 +1,58 @@ +/* + * Copyright 2023 Adobe. All rights reserved. + * This file is licensed to you under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS + * OF ANY KIND, either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ + +import { v4 as uuidv4 } from 'uuid'; + +import { randomDate } from './util.js'; + +function generateRandomAudit(siteId, auditType) { + let auditResult = {}; + const auditedAt = randomDate(new Date(2020, 0, 1), new Date()).toISOString(); + const expiresAt = new Date(auditedAt); + expiresAt.setDate(expiresAt.getDate() + 30); + const fullAuditRef = `s3://audit-results/${uuidv4()}.json`; + + function getRandomDecimal(precision) { + return parseFloat(Math.random().toFixed(precision)); + } + + function getRandomInt(max) { + return Math.floor(Math.random() * max); + } + + if (auditType === 'lhs') { + auditResult = { + performance: getRandomDecimal(2), + accessibility: getRandomDecimal(2), + 'best-practices': getRandomDecimal(2), + SEO: getRandomDecimal(2), + }; + } else if (auditType === 'cwv') { + auditResult = { + LCP: getRandomInt(4000), // LCP in milliseconds + FID: getRandomInt(100), // FID in milliseconds + CLS: getRandomDecimal(2), // CLS score + }; + } + + return { + siteId, + SK: `${auditType}#${auditedAt}`, + auditType, + auditedAt, + auditResult, + expiresAt: Math.floor(expiresAt.getTime() / 1000), // AWS expects unix epoch in seconds + fullAuditRef, + }; +} + +export { generateRandomAudit }; diff --git a/packages/spacecat-shared-data-access/test-it/db.js b/packages/spacecat-shared-data-access/test-it/db.js new file mode 100644 index 000000000..7bff40e77 --- /dev/null +++ b/packages/spacecat-shared-data-access/test-it/db.js @@ -0,0 +1,22 @@ +/* + * Copyright 2023 Adobe. All rights reserved. + * This file is licensed to you under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS + * OF ANY KIND, either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ + +import { DynamoDB } from '@aws-sdk/client-dynamodb'; +import { DynamoDBDocument } from '@aws-sdk/lib-dynamodb'; + +const dbClient = new DynamoDB({ + endpoint: 'http://127.0.0.1:8000', + region: 'local', +}); +const docClient = DynamoDBDocument.from(dbClient); + +export { dbClient, docClient }; diff --git a/packages/spacecat-shared-data-access/test-it/db.test.js b/packages/spacecat-shared-data-access/test-it/db.test.js new file mode 100644 index 000000000..bb43b5b52 --- /dev/null +++ b/packages/spacecat-shared-data-access/test-it/db.test.js @@ -0,0 +1,112 @@ +/* + * Copyright 2023 Adobe. All rights reserved. + * This file is licensed to you under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS + * OF ANY KIND, either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ + +/* eslint-env mocha */ + +import { expect } from 'chai'; + +import dynamoDbLocal from 'dynamo-db-local'; +import { isValidUrl } from '@adobe/spacecat-shared-utils'; + +import { createDataAccess } from '../src/index.js'; +import { AUDIT_TYPE_LHS } from '../src/models/audit.js'; + +import generateSampleData from './generateSampleData.js'; + +async function sleep(ms) { + return new Promise((resolve) => { + setTimeout(resolve, ms); + }); +} + +function checkSite(site) { + expect(site.getId()).to.be.a('string'); + expect(site.getBaseURL()).to.be.a('string'); + expect(site.getImsOrgId()).to.be.a('string'); + expect(site.getCreatedAt()).to.be.a('string'); + expect(site.getUpdatedAt()).to.be.a('string'); + expect(site.getAudits()).to.be.an('array'); +} + +describe('DynamoDB Integration Test', async () => { + let dynamoDbLocalProcess; + let dataAccess; + + const NUMBER_OF_SITES = 10; + const NUMBER_OF_AUDITS_PER_SITE = 3; + + before(async function () { + this.timeout(3000); + + process.env.AWS_REGION = 'local'; + + dynamoDbLocalProcess = dynamoDbLocal.spawn({ port: 8000, sharedDb: true }); + + await sleep(500); + + await generateSampleData(NUMBER_OF_SITES, NUMBER_OF_AUDITS_PER_SITE); + + dataAccess = createDataAccess(console); + }); + + after(() => { + dynamoDbLocalProcess.kill(); + }); + + it('gets sites', async () => { + const sites = await dataAccess.getSites(); + + expect(sites.length).to.equal(NUMBER_OF_SITES); + + sites.forEach((site) => { + checkSite(site); + expect(site.getAudits()).to.be.an('array').that.has.lengthOf(0); + }); + }); + + it('gets sites to audit', async () => { + const sites = await dataAccess.getSitesToAudit(); + + expect(sites.length).to.equal(NUMBER_OF_SITES); + + sites.forEach((baseURL) => { + expect(baseURL).to.be.a('string'); + expect(isValidUrl(baseURL)).to.equal(true); + }); + }); + + it('gets sites with latest audit', async () => { + const sites = await dataAccess.getSitesWithLatestAudit(AUDIT_TYPE_LHS); + + // Every tenth site will not have any audits + expect(sites.length).to.equal(NUMBER_OF_SITES - 1); + + sites.forEach((site) => { + checkSite(site); + expect(site.getAudits()).to.be.an('array').that.has.lengthOf(1); + site.getAudits().forEach((audit) => { + expect(audit.getAuditType()).to.equal(AUDIT_TYPE_LHS); + expect(Object.keys(audit.getScores())).to.have.members( + ['performance', 'seo', 'accessibility', 'best-practices'], + ); + }); + }); + }); + + it('gets site by baseURL', async () => { + const site = await dataAccess.getSiteByBaseURL('https://example1.com'); + + expect(site).to.be.an('object'); + + checkSite(site); + }); +}); diff --git a/packages/spacecat-shared-data-access/test-it/generateSampleData.js b/packages/spacecat-shared-data-access/test-it/generateSampleData.js new file mode 100644 index 000000000..7f74c3b4a --- /dev/null +++ b/packages/spacecat-shared-data-access/test-it/generateSampleData.js @@ -0,0 +1,168 @@ +/* + * Copyright 2023 Adobe. All rights reserved. + * This file is licensed to you under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS + * OF ANY KIND, either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ + +import { v4 as uuidv4 } from 'uuid'; + +import { dbClient, docClient as client } from './db.js'; +import { generateRandomAudit } from './auditUtils.js'; +import { createTable, deleteTable } from './tableOperations.js'; + +import schema from '../docs/schema.json' assert { type: 'json' }; + +/** + * Creates all tables defined in a schema. + * + * Iterates over a predefined schema object and creates each table using the createTable function. + * The schema object should define all required attributes and configurations for each table. + */ +async function createTablesFromSchema() { + const creationPromises = schema.DataModel.map( + (tableDefinition) => createTable(dbClient, tableDefinition), + ); + await Promise.all(creationPromises); +} + +/** + * Deletes a predefined set of tables from the database. + * + * Iterates over a list of table names and deletes each one using the deleteTable function. + * This is typically used to clean up the database before creating new tables or + * generating test data. + */ +async function deleteExistingTables() { + const deletionPromises = ['sites', 'audits', 'latest_audits'] + .map((tableName) => deleteTable(dbClient, tableName)); + await Promise.all(deletionPromises); +} + +/** + * Performs a batch write operation for a specified table in DynamoDB. + * + * @param {string} tableName - The name of the table to perform the batch write operation on. + * @param {Array} items - An array of items to be written to the table. + * + * @example + * // Example usage + * const itemsToWrite = [{ id: '1', data: 'example' }, { id: '2', data: 'sample' }]; + * batchWrite('myTable', itemsToWrite); + */ +async function batchWrite(tableName, items) { + const batchWriteRequests = []; + while (items.length) { + const batch = items.splice(0, 25).map((item) => ({ + PutRequest: { Item: item }, + })); + + batchWriteRequests.push(client.batchWrite({ + RequestItems: { [tableName]: batch }, + })); + } + + await Promise.all(batchWriteRequests); +} + +/** + * Generates audit data for a specific site. + * + * @param {string} siteId - The ID of the site for which to generate audit data. + * @param {Array} auditTypes - An array of audit types to generate data for. + * @param {number} numberOfAuditsPerType - The number of audits to generate for each type. + * @returns {Object} An object containing arrays of audit data and latest audit data for the site. + * + * @example + * // Example usage + * const audits = generateAuditData('site123', ['lhs', 'cwv'], 5); + */ +function generateAuditData(siteId, auditTypes, numberOfAuditsPerType) { + const latestAudits = {}; + const auditData = []; + + for (const type of auditTypes) { + for (let j = 0; j < numberOfAuditsPerType; j += 1) { + const audit = generateRandomAudit(siteId, type); + auditData.push(audit); + + // Update latest audit for each type + if (!latestAudits[type] + || new Date(audit.auditedAt) > new Date(latestAudits[type].auditedAt)) { + latestAudits[type] = audit; + } + } + } + + const latestAuditData = Object.values(latestAudits).map((audit) => { + // Modify the audit data for the latest_audits table + let GSI1SK = `${audit.auditType}#`; + if (audit.auditType === 'lhs') { + GSI1SK += Object.values(audit.auditResult).map((score) => (parseFloat(score) * 100).toFixed(0)).join('#'); + } else { + GSI1SK += Object.values(audit.auditResult).join('#'); + } + + return { + ...audit, + GSI1PK: 'ALL_LATEST_AUDITS', + GSI1SK, + }; + }); + + return { auditData, latestAuditData }; +} + +/** + * Generates sample data for testing purposes. + * + * @param {number} [numberOfSites=10] - The number of sites to generate. + * @param {number} [numberOfAuditsPerType=5] - The number of audits per type to generate + * for each site. + * + * @example + * // Example usage + * generateSampleData(20, 10); // Generates 20 sites with 10 audits per type for each site + */ +export default async function generateSampleData(numberOfSites = 10, numberOfAuditsPerType = 5) { + console.time('Sample data generated in'); + await deleteExistingTables(); + await createTablesFromSchema(); + + const auditTypes = ['lhs', 'cwv']; + const sites = []; + const auditItems = []; + const latestAuditItems = []; + const nowIso = new Date().toISOString(); + + // Generate site data + for (let i = 0; i < numberOfSites; i += 1) { + const siteId = uuidv4(); + sites.push({ + id: siteId, + baseURL: `https://example${i}.com`, + imsOrgId: `${i}-1234@AdobeOrg`, + GSI1PK: 'ALL_SITES', + createdAt: nowIso, + updatedAt: nowIso, + }); + + if (i % 10 !== 0) { // Every tenth site will not have any audits + const latestAudits = generateAuditData(siteId, auditTypes, numberOfAuditsPerType); + auditItems.push(...latestAudits.auditData); + latestAuditItems.push(...latestAudits.latestAuditData); + } + } + + await batchWrite('sites', sites); + await batchWrite('audits', auditItems); + await batchWrite('latest_audits', latestAuditItems); + + console.log(`Generated ${numberOfSites} sites with ${numberOfAuditsPerType} audits per type for each site`); + console.timeEnd('Sample data generated in'); +} diff --git a/packages/spacecat-shared-data-access/test-it/tableOperations.js b/packages/spacecat-shared-data-access/test-it/tableOperations.js new file mode 100644 index 000000000..b7799143d --- /dev/null +++ b/packages/spacecat-shared-data-access/test-it/tableOperations.js @@ -0,0 +1,161 @@ +/* + * Copyright 2023 Adobe. All rights reserved. + * This file is licensed to you under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS + * OF ANY KIND, either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ + +import { CreateTableCommand, DeleteTableCommand } from '@aws-sdk/client-dynamodb'; + +/** + * Creates a DynamoDB table based on the provided table definition. + * + * The function defines key schema and attribute definitions for the table, + * including the partition key and optional sort key. It also handles the + * configuration of global secondary indexes (GSIs) if provided + * in the table definition. + * + * @param {Object} dbClient - The DynamoDB client instance used for sending commands. + * @param {Object} tableDefinition - An object describing the table to be created. It should contain + * the table name, key attributes, and optionally GSIs. + * + * @example + * // Example of tableDefinition object + * { + * TableName: 'MyTable', + * KeyAttributes: { + * PartitionKey: { AttributeName: 'Id', AttributeType: 'S' }, + * SortKey: { AttributeName: 'SortKey', AttributeType: 'N' } + * }, + * GlobalSecondaryIndexes: [ + * { + * IndexName: 'MyGSI', + * KeyAttributes: { + * PartitionKey: { AttributeName: 'GSIKey', AttributeType: 'S' }, + * SortKey: { AttributeName: 'GSISortKey', AttributeType: 'N' } + * }, + * Projection: { + * ProjectionType: 'ALL' + * } + * } + * ] + * } + */ +async function createTable(dbClient, tableDefinition) { + const keySchema = []; + const attributeDefinitions = []; + + // Define partition key + if (tableDefinition.KeyAttributes.PartitionKey) { + keySchema.push({ AttributeName: tableDefinition.KeyAttributes.PartitionKey.AttributeName, KeyType: 'HASH' }); + attributeDefinitions.push({ + AttributeName: tableDefinition.KeyAttributes.PartitionKey.AttributeName, + AttributeType: tableDefinition.KeyAttributes.PartitionKey.AttributeType, + }); + } + + // Define sort key if present + if (tableDefinition.KeyAttributes.SortKey) { + keySchema.push({ AttributeName: tableDefinition.KeyAttributes.SortKey.AttributeName, KeyType: 'RANGE' }); + attributeDefinitions.push({ + AttributeName: tableDefinition.KeyAttributes.SortKey.AttributeName, + AttributeType: tableDefinition.KeyAttributes.SortKey.AttributeType, + }); + } + + const params = { + TableName: tableDefinition.TableName, + KeySchema: keySchema, + AttributeDefinitions: attributeDefinitions, + BillingMode: 'PAY_PER_REQUEST', // or specify ProvisionedThroughput + }; + + // Add GSI configuration if present + if (tableDefinition.GlobalSecondaryIndexes) { + params.GlobalSecondaryIndexes = tableDefinition.GlobalSecondaryIndexes.map((gsi) => { + // Add GSI key attributes to AttributeDefinitions + if (gsi.KeyAttributes.PartitionKey) { + if (!attributeDefinitions.some( + (attr) => attr.AttributeName === gsi.KeyAttributes.PartitionKey.AttributeName, + ) + ) { + attributeDefinitions.push({ + AttributeName: gsi.KeyAttributes.PartitionKey.AttributeName, + AttributeType: gsi.KeyAttributes.PartitionKey.AttributeType, + }); + } + } + if (gsi.KeyAttributes.SortKey) { + if (!attributeDefinitions.some( + (attr) => attr.AttributeName === gsi.KeyAttributes.SortKey.AttributeName, + ) + ) { + attributeDefinitions.push({ + AttributeName: gsi.KeyAttributes.SortKey.AttributeName, + AttributeType: gsi.KeyAttributes.SortKey.AttributeType, + }); + } + } + + // Define GSI Key Schema + const gsiKeySchema = [ + { AttributeName: gsi.KeyAttributes.PartitionKey.AttributeName, KeyType: 'HASH' }, + gsi.KeyAttributes.SortKey ? { + AttributeName: gsi.KeyAttributes.SortKey.AttributeName, + KeyType: 'RANGE', + } : null, + ].filter(Boolean); + + return { + IndexName: gsi.IndexName, + KeySchema: gsiKeySchema, + Projection: gsi.Projection, + }; + }); + } + + try { + await dbClient.send(new CreateTableCommand(params)); + console.log(`Table ${tableDefinition.TableName} created successfully.`); + } catch (error) { + console.error(`Error creating table ${tableDefinition.TableName}:`, error); + } +} + +/** + * Deletes a specified DynamoDB table. + * + * The function sends a command to delete the table with the given table name. + * It handles the response and logs the result of the operation, including handling the case + * where the table does not exist. + * + * @param {Object} dbClient - The DynamoDB client instance used for sending commands. + * @param {string} tableName - The name of the table to be deleted. + * + * @example + * // Example usage + * deleteTable(dynamoDBClient, 'MyTable'); + */ +async function deleteTable(dbClient, tableName) { + const deleteParams = { + TableName: tableName, + }; + + try { + await dbClient.send(new DeleteTableCommand(deleteParams)); + console.log(`Table ${tableName} deleted successfully.`); + } catch (error) { + if (error.name === 'ResourceNotFoundException') { + console.log(`Table ${tableName} does not exist.`); + } else { + console.error(`Error deleting table ${tableName}:`, error); + } + } +} + +export { createTable, deleteTable }; diff --git a/packages/spacecat-shared-data-access/test-it/util.js b/packages/spacecat-shared-data-access/test-it/util.js new file mode 100644 index 000000000..2d49f7274 --- /dev/null +++ b/packages/spacecat-shared-data-access/test-it/util.js @@ -0,0 +1,22 @@ +/* + * Copyright 2023 Adobe. All rights reserved. + * This file is licensed to you under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS + * OF ANY KIND, either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ +const randomDate = (start, end) => new Date( + start.getTime() + Math.random() * (end.getTime() - start.getTime()), +); + +// Generates a random decimal number with given precision +const getRandomDecimal = (precision) => parseFloat(Math.random().toFixed(precision)); + +// Generates a random integer up to a given maximum +const getRandomInt = (max) => Math.floor(Math.random() * max); + +export { randomDate, getRandomDecimal, getRandomInt }; diff --git a/packages/spacecat-shared-data-access/test/models/audit.test.js b/packages/spacecat-shared-data-access/test/models/audit.test.js index f6d48683e..477df696b 100644 --- a/packages/spacecat-shared-data-access/test/models/audit.test.js +++ b/packages/spacecat-shared-data-access/test/models/audit.test.js @@ -15,7 +15,6 @@ import { expect } from 'chai'; import { createAudit } from '../../src/models/audit.js'; -// Constants for testing const validData = { siteId: '123', auditedAt: new Date().toISOString(), diff --git a/packages/spacecat-shared-data-access/test/sites/index.test.js b/packages/spacecat-shared-data-access/test/sites/index.test.js index 0774d2516..804d21e8f 100644 --- a/packages/spacecat-shared-data-access/test/sites/index.test.js +++ b/packages/spacecat-shared-data-access/test/sites/index.test.js @@ -138,27 +138,27 @@ describe('Site Index Tests', () => { it('calls getSiteByBaseURL and returns an array/object', async () => { const result = await exportedFunctions.getSiteByBaseURL(); expect(result).to.be.null; - expect(mockDynamoClient.getItem.called).to.be.true; + expect(mockDynamoClient.query.called).to.be.true; }); it('calls getSiteByBaseURLWithAuditInfo and returns an array/object', async () => { const result = await exportedFunctions.getSiteByBaseURLWithAuditInfo(); expect(result).to.be.null; - expect(mockDynamoClient.getItem.called).to.be.true; + expect(mockDynamoClient.query.called).to.be.true; }); it('calls getSiteByBaseURLWithAuditInfo and returns null when site is undefined', async () => { - mockDynamoClient.query.resolves(undefined); + mockDynamoClient.query.resolves([]); const result = await exportedFunctions.getSiteByBaseURLWithAuditInfo('baseUrl', 'auditType'); expect(result).to.be.null; }); it('calls getSiteByBaseURLWithAuditInfo and assigns latest audit when latestOnly is true', async () => { - const mockSiteData = { + const mockSiteData = [{ id: 'site1', baseURL: 'https://example.com', - }; + }]; const mockLatestAuditData = [{ siteId: 'site1', @@ -168,8 +168,8 @@ describe('Site Index Tests', () => { fullAuditRef: 'https://example.com', }]; - mockDynamoClient.getItem.onFirstCall().resolves(mockSiteData); - mockDynamoClient.query.onFirstCall().resolves(mockLatestAuditData); + mockDynamoClient.query.onFirstCall().resolves(mockSiteData); + mockDynamoClient.query.onSecondCall().resolves(mockLatestAuditData); const result = await exportedFunctions.getSiteByBaseURLWithAuditInfo('https://example.com', 'type1', true); const audits = result.getAudits(); @@ -185,10 +185,10 @@ describe('Site Index Tests', () => { }); it('calls getSiteByBaseURLWithAuditInfo and assigns all audits when latestOnly is false', async () => { - const mockSiteData = { + const mockSiteData = [{ id: 'site1', baseURL: 'https://example.com', - }; + }]; const mockLatestAuditData = [{ siteId: 'site1', @@ -205,8 +205,8 @@ describe('Site Index Tests', () => { fullAuditRef: 'https://example2.com', }]; - mockDynamoClient.getItem.onFirstCall().resolves(mockSiteData); - mockDynamoClient.query.onFirstCall().resolves(mockLatestAuditData); + mockDynamoClient.query.onFirstCall().resolves(mockSiteData); + mockDynamoClient.query.onSecondCall().resolves(mockLatestAuditData); const result = await exportedFunctions.getSiteByBaseURLWithAuditInfo('baseUrl', 'auditType', false); const audits = result.getAudits(); @@ -228,19 +228,19 @@ describe('Site Index Tests', () => { it('calls getSiteByBaseURLWithAudits and returns an array/object', async () => { const result = await exportedFunctions.getSiteByBaseURLWithAudits(); expect(result).to.be.null; - expect(mockDynamoClient.getItem.called).to.be.true; + expect(mockDynamoClient.query.called).to.be.true; }); it('calls getSiteByBaseURLWithLatestAudit and returns an array/object', async () => { const result = await exportedFunctions.getSiteByBaseURLWithLatestAudit(); expect(result).to.be.null; - expect(mockDynamoClient.getItem.called).to.be.true; + expect(mockDynamoClient.query.called).to.be.true; }); describe('addSite Tests', () => { beforeEach(() => { mockDynamoClient = { - getItem: sinon.stub().returns(Promise.resolve(null)), + query: sinon.stub().returns(Promise.resolve([])), putItem: sinon.stub().returns(Promise.resolve()), }; mockLog = { log: sinon.stub() }; @@ -253,12 +253,12 @@ describe('Site Index Tests', () => { expect(mockDynamoClient.putItem.calledOnce).to.be.true; expect(result.getBaseURL()).to.equal(siteData.baseURL); expect(result.getId()).to.be.a('string'); - expect(result.getAudits()).to.be.an('object').that.is.empty; + expect(result.getAudits()).to.be.an('array').that.is.empty; }); it('throws an error if site already exists', async () => { const siteData = { baseURL: 'https://existingsite.com' }; - mockDynamoClient.getItem.returns(Promise.resolve(siteData)); + mockDynamoClient.query.returns(Promise.resolve([siteData])); await expect(exportedFunctions.addSite(siteData)).to.be.rejectedWith('Site already exists'); }); @@ -272,7 +272,7 @@ describe('Site Index Tests', () => { beforeEach(() => { mockDynamoClient = { - getItem: sinon.stub().returns(Promise.resolve(null)), + query: sinon.stub().returns(Promise.resolve([])), putItem: sinon.stub().returns(Promise.resolve()), }; mockLog = { log: sinon.stub() }; @@ -281,7 +281,7 @@ describe('Site Index Tests', () => { it('updates an existing site successfully', async () => { const siteData = { baseURL: 'https://existingsite.com' }; - mockDynamoClient.getItem.returns(Promise.resolve(siteData)); + mockDynamoClient.query.returns(Promise.resolve([siteData])); const site = await exportedFunctions.getSiteByBaseURL(siteData.baseURL); site.updateBaseURL('https://newsite.com'); From 0d04ae86ec1bc5b3ab89be411a0228c3a8f6d187 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominique=20J=C3=A4ggi?= Date: Thu, 30 Nov 2023 18:03:08 +0100 Subject: [PATCH 08/28] chore: doc --- docs/API.md | 65 +++++++++++---- .../spacecat-shared-data-access/README.md | 80 ++++++++++++++++++- 2 files changed, 130 insertions(+), 15 deletions(-) diff --git a/docs/API.md b/docs/API.md index 6392f8287..32227de18 100644 --- a/docs/API.md +++ b/docs/API.md @@ -1,11 +1,23 @@ +## Constants + +
+
createDataAccess ⇒ object
+

Creates a data access object.

+
+
+ ## Functions
createClient(log, dbClient, docClient) ⇒ Object

Creates a client object for interacting with DynamoDB.

+
isArray(value) ⇒ boolean
+

Determines if the given parameter is an array.

+
isBoolean(value) ⇒ boolean
-

Determines if the given value is a boolean or a string representation of a boolean.

+

Determines case-insensitively if the given value is a boolean or a string +representation of a boolean.

isInteger(value) ⇒ boolean

Checks if the given value is an integer.

@@ -13,16 +25,16 @@
isNumber(value) ⇒ boolean

Determines if the given value is a number.

-
isObject(obj) ⇒ boolean
+
isObject(value) ⇒ boolean

Checks if the given parameter is an object and not an array or null.

-
isString(str) ⇒ boolean
+
isString(value) ⇒ boolean

Determines if the given parameter is a string.

hasText(str) ⇒ boolean

Checks if the given string is not empty.

-
isValidDate(obj) ⇒ boolean
+
isValidDate(value) ⇒ boolean

Checks whether the given object is a valid JavaScript Date.

isIsoDate(str) ⇒ boolean
@@ -41,10 +53,22 @@ following UTC time offsets format.

Converts a given value to a boolean. Throws an error if the value is not a boolean.

arrayEquals(a, b) ⇒ boolean
-

Compares two arrays for equality.

+

Compares two arrays for equality. Supports primitive array item types only.

+ + +## createDataAccess ⇒ object +Creates a data access object. + +**Kind**: global constant +**Returns**: object - data access object + +| Param | Type | Description | +| --- | --- | --- | +| log | Logger | logger | + ## createClient(log, dbClient, docClient) ⇒ Object @@ -57,12 +81,25 @@ Creates a client object for interacting with DynamoDB. | --- | --- | --- | | log | Object | The logging object, defaults to console. | | dbClient | DynamoDB | The AWS SDK DynamoDB client instance. | -| docClient | DynamoDBDocumentClient | The AWS SDK DynamoDB Document client instance. | +| docClient | DynamoDBDocument | The AWS SDK DynamoDB Document client instance. | + + + +## isArray(value) ⇒ boolean +Determines if the given parameter is an array. + +**Kind**: global function +**Returns**: boolean - True if the parameter is an array, false otherwise. + +| Param | Type | Description | +| --- | --- | --- | +| value | \* | The value to check. | ## isBoolean(value) ⇒ boolean -Determines if the given value is a boolean or a string representation of a boolean. +Determines case-insensitively if the given value is a boolean or a string +representation of a boolean. **Kind**: global function **Returns**: boolean - True if the value is a boolean or a string representation of a boolean. @@ -97,7 +134,7 @@ Determines if the given value is a number. -## isObject(obj) ⇒ boolean +## isObject(value) ⇒ boolean Checks if the given parameter is an object and not an array or null. **Kind**: global function @@ -105,11 +142,11 @@ Checks if the given parameter is an object and not an array or null. | Param | Type | Description | | --- | --- | --- | -| obj | \* | The object to check. | +| value | \* | The value to check. | -## isString(str) ⇒ boolean +## isString(value) ⇒ boolean Determines if the given parameter is a string. **Kind**: global function @@ -117,7 +154,7 @@ Determines if the given parameter is a string. | Param | Type | Description | | --- | --- | --- | -| str | \* | The string to check. | +| value | \* | The value to check. | @@ -133,7 +170,7 @@ Checks if the given string is not empty. -## isValidDate(obj) ⇒ boolean +## isValidDate(value) ⇒ boolean Checks whether the given object is a valid JavaScript Date. **Kind**: global function @@ -141,7 +178,7 @@ Checks whether the given object is a valid JavaScript Date. | Param | Type | Description | | --- | --- | --- | -| obj | \* | The object to check. | +| value | \* | The value to check. | @@ -201,7 +238,7 @@ Converts a given value to a boolean. Throws an error if the value is not a boole ## arrayEquals(a, b) ⇒ boolean -Compares two arrays for equality. +Compares two arrays for equality. Supports primitive array item types only. **Kind**: global function **Returns**: boolean - True if the arrays are equal, false otherwise. diff --git a/packages/spacecat-shared-data-access/README.md b/packages/spacecat-shared-data-access/README.md index e23e5be9e..588bf8b79 100644 --- a/packages/spacecat-shared-data-access/README.md +++ b/packages/spacecat-shared-data-access/README.md @@ -1,2 +1,80 @@ -# Data Access Module +# SpaceCat Shared Data Access +This Node.js module, `spacecat-shared-data-access`, is a comprehensive data access layer for managing sites and their audits, leveraging Amazon DynamoDB. It's tailored for the `StarCatalogue` model, ensuring efficient querying and robust data manipulation. + +## Installation + +```bash +npm install @adobe/spacecat-shared-data-access +``` + +## Entities + +### Sites +- **id** (String): Unique identifier for a site. +- **baseURL** (String): Base URL of the site. +- **imsOrgId** (String): Organization ID associated with the site. +- **createdAt** (String): Timestamp of creation. +- **updatedAt** (String): Timestamp of the last update. +- **GSI1PK** (String): Partition key for the Global Secondary Index. + +### Audits +- **siteId** (String): Identifier of the site being audited. +- **SK** (String): Sort key, typically a composite of audit type and timestamp. +- **auditedAt** (String): Timestamp of the audit. +- **auditResult** (Map): Results of the audit. +- **auditType** (String): Type of the audit. +- **expiresAt** (Number): Expiry timestamp of the audit. +- **fullAuditRef** (String): Reference to the full audit details. + +## DynamoDB Data Model + +The module is designed to work with the following DynamoDB tables: + +1. **Sites Table**: Manages site records. +2. **Audits Table**: Stores audit information for each site. +3. **Latest Audits Table**: Holds only the latest audit for each site for quick access. + +Each table is designed with scalability and efficient querying in mind, utilizing both key and non-key attributes effectively. + +For a detailed schema, refer to `docs/schema.json`. This schema is importable to Amazon NoSQL Workbench and used by the integration tests. + +## Integration Testing + +The module includes comprehensive integration tests embedding a local DynamoDB server with in-memory storage for testing: + +```bash +npm run test:it +``` + +These tests create the schema, generate sample data, and test the data access patterns against the local DynamoDB instance. + +## Data Access API + +The module provides two main DAOs: + +### Site Functions +- `getSites` +- `getSitesToAudit` +- `getSitesWithLatestAudit` +- `getSiteByBaseURL` +- `getSiteByBaseURLWithAuditInfo` +- `getSiteByBaseURLWithAudits` +- `getSiteByBaseURLWithLatestAudit` +- `addSite` +- `updateSite` + +### Audit Functions +- `getAuditsForSite` +- `getAuditForSite` +- `getLatestAudits` +- `getLatestAuditForSite` +- `addAudit` + +## Contributing + +Contributions to `spacecat-shared-data-access` are welcome. Please adhere to the standard Git workflow and submit pull requests for proposed changes. + +## License + +Licensed under the Apache-2.0 License. From a27ccb2cfc88b860d36a63d16db992a5b829a928 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominique=20J=C3=A4ggi?= Date: Thu, 30 Nov 2023 18:06:41 +0100 Subject: [PATCH 09/28] fix: verify start before end date --- packages/spacecat-shared-data-access/test-it/util.js | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/packages/spacecat-shared-data-access/test-it/util.js b/packages/spacecat-shared-data-access/test-it/util.js index 2d49f7274..c2ce6fc12 100644 --- a/packages/spacecat-shared-data-access/test-it/util.js +++ b/packages/spacecat-shared-data-access/test-it/util.js @@ -9,9 +9,14 @@ * OF ANY KIND, either express or implied. See the License for the specific language * governing permissions and limitations under the License. */ -const randomDate = (start, end) => new Date( - start.getTime() + Math.random() * (end.getTime() - start.getTime()), -); +const randomDate = (start, end) => { + if (start.getTime() >= end.getTime()) { + throw new Error('start must be before end'); + } + return new Date( + start.getTime() + Math.random() * (end.getTime() - start.getTime()), + ); +}; // Generates a random decimal number with given precision const getRandomDecimal = (precision) => parseFloat(Math.random().toFixed(precision)); From 2047fbfffb40302222a2e8e3b654cf1e8c0fc5e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominique=20J=C3=A4ggi?= Date: Thu, 30 Nov 2023 18:23:20 +0100 Subject: [PATCH 10/28] fix: add audit result validation, tests --- .../src/models/audit.js | 45 ++++++++++++------- .../test/audits/index.test.js | 40 +++++++++++++++-- .../test/models/audit.test.js | 11 +++-- .../test/models/base.test.js | 2 +- .../test/models/site.test.js | 2 +- .../test/sites/index.test.js | 44 +++++++++++++----- 6 files changed, 106 insertions(+), 38 deletions(-) diff --git a/packages/spacecat-shared-data-access/src/models/audit.js b/packages/spacecat-shared-data-access/src/models/audit.js index 6f603c1b5..82cd68dc3 100644 --- a/packages/spacecat-shared-data-access/src/models/audit.js +++ b/packages/spacecat-shared-data-access/src/models/audit.js @@ -17,6 +17,31 @@ export const AUDIT_TYPE_LHS = 'lhs'; const EXPIRES_IN_DAYS = 30; +const AUDIT_TYPE_PROPERTIES = { + [AUDIT_TYPE_LHS]: ['performance', 'seo', 'accessibility', 'best-practices'], +}; + +/** + * Validates if the auditResult contains the required properties for the given audit type. + * @param {object} auditResult - The audit result to validate. + * @param {string} auditType - The type of the audit. + * @returns {boolean} - True if valid, false otherwise. + */ +const validateAuditResult = (auditResult, auditType) => { + const expectedProperties = AUDIT_TYPE_PROPERTIES[auditType]; + if (!expectedProperties) { + throw new Error(`Unknown audit type: ${auditType}`); + } + + for (const prop of expectedProperties) { + if (!(prop in auditResult)) { + throw new Error(`Missing expected property '${prop}' for audit type '${auditType}'`); + } + } + + return true; +}; + /** * Creates a new Audit. * @param {object } data - audit data @@ -31,23 +56,7 @@ const Audit = (data = {}) => { self.getAuditType = () => self.state.auditType.toLowerCase(); self.getExpiresAt = () => self.state.expiresAt; self.getFullAuditRef = () => self.state.fullAuditRef; - self.getScores = () => { - const auditResult = self.getAuditResult(); - - if (self.getAuditType() === AUDIT_TYPE_LHS) { - const { - performance, seo, accessibility, 'best-practices': bestPractices, - } = auditResult; - return { - performance, - seo, - accessibility, - 'best-practices': bestPractices, - }; - } - - return auditResult; - }; + self.getScores = () => self.getAuditResult(); return Object.freeze(self); }; @@ -77,6 +86,8 @@ export const createAudit = (data) => { throw new Error('Audit result must be an object'); } + validateAuditResult(data.auditResult, data.auditType); + if (!hasText(newState.fullAuditRef)) { throw new Error('Full audit ref must be provided'); } diff --git a/packages/spacecat-shared-data-access/test/audits/index.test.js b/packages/spacecat-shared-data-access/test/audits/index.test.js index 8c689596e..ce3cf4c3a 100644 --- a/packages/spacecat-shared-data-access/test/audits/index.test.js +++ b/packages/spacecat-shared-data-access/test/audits/index.test.js @@ -17,7 +17,7 @@ import sinon from 'sinon'; import { auditFunctions } from '../../src/audits/index.js'; -describe('Audit Index Tests', () => { +describe('Audit Access Pattern Tests', () => { describe('Audit Functions Export Tests', () => { const mockDynamoClient = {}; const mockLog = {}; @@ -89,9 +89,14 @@ describe('Audit Index Tests', () => { it('successfully retrieves an audit for a site', async () => { const mockAuditData = [{ siteId: 'siteId', - auditType: 'type1', + auditType: 'lhs', auditedAt: new Date().toISOString(), - auditResult: { score: 1 }, + auditResult: { + performance: 0.9, + seo: 0.9, + accessibility: 0.9, + 'best-practices': 0.9, + }, fullAuditRef: 'https://someurl.com', }]; mockDynamoClient.query.returns(Promise.resolve(mockAuditData)); @@ -120,7 +125,12 @@ describe('Audit Index Tests', () => { siteId: 'siteId', auditType: 'lhs', auditedAt: new Date().toISOString(), - auditResult: { score: 1 }, + auditResult: { + performance: 0.9, + seo: 0.9, + accessibility: 0.9, + 'best-practices': 0.9, + }, fullAuditRef: 'https://someurl.com', }; @@ -150,5 +160,27 @@ describe('Audit Index Tests', () => { await expect(exportedFunctions.addAudit(auditData)).to.be.rejectedWith('Audit already exists'); }); + + it('throws an error for unknown audit type', async () => { + const invalidAuditData = { + ...auditData, + auditType: 'unknownType', // An unknown audit type + }; + + await expect(exportedFunctions.addAudit(invalidAuditData)).to.be.rejectedWith('Unknown audit type'); + }); + + it('throws an error if an expected property is missing in audit results', async () => { + const incompleteAuditData = { + ...auditData, + auditResult: { + performance: 0.9, + seo: 0.9, + // 'accessibility' and 'best-practices' are missing + }, + }; + + await expect(exportedFunctions.addAudit(incompleteAuditData)).to.be.rejectedWith('Missing expected property'); + }); }); }); diff --git a/packages/spacecat-shared-data-access/test/models/audit.test.js b/packages/spacecat-shared-data-access/test/models/audit.test.js index 477df696b..a5fa9658d 100644 --- a/packages/spacecat-shared-data-access/test/models/audit.test.js +++ b/packages/spacecat-shared-data-access/test/models/audit.test.js @@ -18,12 +18,17 @@ import { createAudit } from '../../src/models/audit.js'; const validData = { siteId: '123', auditedAt: new Date().toISOString(), - auditType: 'Type', - auditResult: {}, + auditType: 'lhs', + auditResult: { + performance: 0.9, + seo: 0.9, + accessibility: 0.9, + 'best-practices': 0.9, + }, fullAuditRef: 'ref123', }; -describe('Audit Module Tests', () => { +describe('Audit Model Tests', () => { describe('Validation Tests', () => { it('throws an error if siteId is not provided', () => { expect(() => createAudit({ ...validData, siteId: '' })).to.throw('Site ID must be provided'); diff --git a/packages/spacecat-shared-data-access/test/models/base.test.js b/packages/spacecat-shared-data-access/test/models/base.test.js index 0b2759434..8d320cee3 100644 --- a/packages/spacecat-shared-data-access/test/models/base.test.js +++ b/packages/spacecat-shared-data-access/test/models/base.test.js @@ -15,7 +15,7 @@ import { expect } from 'chai'; import { Base } from '../../src/models/base.js'; -describe('Base Entity Tests', () => { +describe('Base Model Tests', () => { describe('Initialization Tests', () => { it('should automatically assign a UUID if no id is provided', () => { const baseEntity = Base(); diff --git a/packages/spacecat-shared-data-access/test/models/site.test.js b/packages/spacecat-shared-data-access/test/models/site.test.js index a195af4ff..4d1dd5b63 100644 --- a/packages/spacecat-shared-data-access/test/models/site.test.js +++ b/packages/spacecat-shared-data-access/test/models/site.test.js @@ -21,7 +21,7 @@ const validData = { imsOrgId: 'org123', }; -describe('Site Module Tests', () => { +describe('Site Model Tests', () => { describe('Validation Tests', () => { it('throws an error if baseURL is not a valid URL', () => { expect(() => createSite({ ...validData, baseURL: 'invalid-url' })).to.throw('Base URL must be a valid URL'); diff --git a/packages/spacecat-shared-data-access/test/sites/index.test.js b/packages/spacecat-shared-data-access/test/sites/index.test.js index 804d21e8f..51376bb80 100644 --- a/packages/spacecat-shared-data-access/test/sites/index.test.js +++ b/packages/spacecat-shared-data-access/test/sites/index.test.js @@ -23,7 +23,7 @@ chai.use(chaiAsPromised); const { expect } = chai; -describe('Site Index Tests', () => { +describe('Site Access Pattern Tests', () => { describe('Site Functions Export Tests', () => { const mockDynamoClient = {}; const mockLog = {}; @@ -107,16 +107,21 @@ describe('Site Index Tests', () => { const mockAuditData = [{ siteId: 'site1', - auditType: 'type1', + auditType: 'lhs', auditedAt: new Date().toISOString(), - auditResult: {}, + auditResult: { + performance: 0.9, + seo: 0.9, + accessibility: 0.9, + 'best-practices': 0.9, + }, fullAuditRef: 'https://example.com', }]; mockDynamoClient.query.onFirstCall().resolves(mockSiteData); mockDynamoClient.query.onSecondCall().resolves(mockAuditData); - const result = await exportedFunctions.getSitesWithLatestAudit('auditType'); + const result = await exportedFunctions.getSitesWithLatestAudit('lhs'); expect(result).to.be.an('array').that.has.lengthOf(1); }); @@ -162,16 +167,21 @@ describe('Site Index Tests', () => { const mockLatestAuditData = [{ siteId: 'site1', - auditType: 'type1', + auditType: 'lhs', auditedAt: new Date().toISOString(), - auditResult: {}, + auditResult: { + performance: 0.9, + seo: 0.9, + accessibility: 0.9, + 'best-practices': 0.9, + }, fullAuditRef: 'https://example.com', }]; mockDynamoClient.query.onFirstCall().resolves(mockSiteData); mockDynamoClient.query.onSecondCall().resolves(mockLatestAuditData); - const result = await exportedFunctions.getSiteByBaseURLWithAuditInfo('https://example.com', 'type1', true); + const result = await exportedFunctions.getSiteByBaseURLWithAuditInfo('https://example.com', 'lhs', true); const audits = result.getAudits(); expect(audits).to.be.an('array').with.lengthOf(1); @@ -192,23 +202,33 @@ describe('Site Index Tests', () => { const mockLatestAuditData = [{ siteId: 'site1', - auditType: 'type1', + auditType: 'lhs', auditedAt: new Date().toISOString(), - auditResult: {}, + auditResult: { + performance: 0.9, + seo: 0.9, + accessibility: 0.9, + 'best-practices': 0.9, + }, fullAuditRef: 'https://example.com', }, { siteId: 'site1', - auditType: 'type2', + auditType: 'lhs', auditedAt: new Date().toISOString(), - auditResult: {}, + auditResult: { + performance: 0.9, + seo: 0.9, + accessibility: 0.9, + 'best-practices': 0.9, + }, fullAuditRef: 'https://example2.com', }]; mockDynamoClient.query.onFirstCall().resolves(mockSiteData); mockDynamoClient.query.onSecondCall().resolves(mockLatestAuditData); - const result = await exportedFunctions.getSiteByBaseURLWithAuditInfo('baseUrl', 'auditType', false); + const result = await exportedFunctions.getSiteByBaseURLWithAuditInfo('baseUrl', 'lhs', false); const audits = result.getAudits(); expect(audits).to.be.an('array').with.lengthOf(2); From dfbcba18599ed4ace9afc7aceb46cffa8433b945 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominique=20J=C3=A4ggi?= Date: Thu, 30 Nov 2023 18:43:08 +0100 Subject: [PATCH 11/28] fix: set updated and created at, add touch --- .../src/models/base.js | 8 ++++- .../src/models/site.js | 4 +++ .../test-it/auditUtils.js | 2 +- .../test-it/db.test.js | 7 +---- .../test/models/base.test.js | 31 +++++++++++++------ .../test/models/site.test.js | 27 ++++++++++++++++ .../spacecat-shared-data-access/test/util.js | 17 ++++++++++ 7 files changed, 79 insertions(+), 17 deletions(-) create mode 100644 packages/spacecat-shared-data-access/test/util.js diff --git a/packages/spacecat-shared-data-access/src/models/base.js b/packages/spacecat-shared-data-access/src/models/base.js index e75dd979f..fa4787cd6 100644 --- a/packages/spacecat-shared-data-access/src/models/base.js +++ b/packages/spacecat-shared-data-access/src/models/base.js @@ -21,16 +21,22 @@ import { isString } from '@adobe/spacecat-shared-utils'; */ export const Base = (data = {}) => { const self = { state: { ...data } }; - const newRecord = !isString(self.state.id); + const nowISO = new Date().toISOString(); if (newRecord) { self.state.id = uuidv4(); + self.state.createdAt = nowISO; + self.state.updatedAt = nowISO; } self.getId = () => self.state.id; self.getCreatedAt = () => self.state.createdAt; self.getUpdatedAt = () => self.state.updatedAt; + self.touch = () => { + self.state.updatedAt = new Date().toISOString(); + }; + return self; }; diff --git a/packages/spacecat-shared-data-access/src/models/site.js b/packages/spacecat-shared-data-access/src/models/site.js index 10eee5201..14cca4c21 100644 --- a/packages/spacecat-shared-data-access/src/models/site.js +++ b/packages/spacecat-shared-data-access/src/models/site.js @@ -32,6 +32,8 @@ const Site = (data = {}) => { } self.state.baseURL = baseURL; + self.touch(); + return self; }; @@ -41,6 +43,8 @@ const Site = (data = {}) => { } self.state.imsOrgId = imsOrgId; + self.touch(); + return self; }; diff --git a/packages/spacecat-shared-data-access/test-it/auditUtils.js b/packages/spacecat-shared-data-access/test-it/auditUtils.js index 9b0babd82..9d1170036 100644 --- a/packages/spacecat-shared-data-access/test-it/auditUtils.js +++ b/packages/spacecat-shared-data-access/test-it/auditUtils.js @@ -32,9 +32,9 @@ function generateRandomAudit(siteId, auditType) { if (auditType === 'lhs') { auditResult = { performance: getRandomDecimal(2), + seo: getRandomDecimal(2), accessibility: getRandomDecimal(2), 'best-practices': getRandomDecimal(2), - SEO: getRandomDecimal(2), }; } else if (auditType === 'cwv') { auditResult = { diff --git a/packages/spacecat-shared-data-access/test-it/db.test.js b/packages/spacecat-shared-data-access/test-it/db.test.js index bb43b5b52..30258dff2 100644 --- a/packages/spacecat-shared-data-access/test-it/db.test.js +++ b/packages/spacecat-shared-data-access/test-it/db.test.js @@ -17,17 +17,12 @@ import { expect } from 'chai'; import dynamoDbLocal from 'dynamo-db-local'; import { isValidUrl } from '@adobe/spacecat-shared-utils'; +import { sleep } from '../test/util.js'; import { createDataAccess } from '../src/index.js'; import { AUDIT_TYPE_LHS } from '../src/models/audit.js'; import generateSampleData from './generateSampleData.js'; -async function sleep(ms) { - return new Promise((resolve) => { - setTimeout(resolve, ms); - }); -} - function checkSite(site) { expect(site.getId()).to.be.a('string'); expect(site.getBaseURL()).to.be.a('string'); diff --git a/packages/spacecat-shared-data-access/test/models/base.test.js b/packages/spacecat-shared-data-access/test/models/base.test.js index 8d320cee3..d4feb48d1 100644 --- a/packages/spacecat-shared-data-access/test/models/base.test.js +++ b/packages/spacecat-shared-data-access/test/models/base.test.js @@ -12,8 +12,11 @@ /* eslint-env mocha */ +import { isIsoDate } from '@adobe/spacecat-shared-utils'; + import { expect } from 'chai'; import { Base } from '../../src/models/base.js'; +import { sleep } from '../util.js'; describe('Base Model Tests', () => { describe('Initialization Tests', () => { @@ -30,26 +33,36 @@ describe('Base Model Tests', () => { }); describe('Getter Method Tests', () => { - it('should correctly return the createdAt date if provided', () => { + it('correctly returns the createdAt date if provided', () => { const createdAt = new Date().toISOString(); const baseEntity = Base({ createdAt }); expect(baseEntity.getCreatedAt()).to.equal(createdAt); }); - it('should return undefined for createdAt if not provided', () => { - const baseEntity = Base(); - expect(baseEntity.getCreatedAt()).to.be.undefined; - }); - - it('should correctly return the updatedAt date if provided', () => { + it('correctly returns the updatedAt date if provided', () => { const updatedAt = new Date().toISOString(); const baseEntity = Base({ updatedAt }); expect(baseEntity.getUpdatedAt()).to.equal(updatedAt); }); + }); - it('should return undefined for updatedAt if not provided', () => { + describe('Timestamp Tests', () => { + it('should set createdAt and updatedAt for new records', () => { const baseEntity = Base(); - expect(baseEntity.getUpdatedAt()).to.be.undefined; + expect(isIsoDate(baseEntity.getCreatedAt())).to.be.true; + expect(isIsoDate(baseEntity.getUpdatedAt())).to.be.true; + expect(baseEntity.getCreatedAt()).to.equal(baseEntity.getUpdatedAt()); + }); + + it('should update updatedAt using touch method', async () => { + const baseEntity = Base(); + const initialUpdatedAt = baseEntity.getUpdatedAt(); + + await sleep(10); + + baseEntity.touch(); + + expect(baseEntity.getUpdatedAt()).to.not.equal(initialUpdatedAt); }); }); }); diff --git a/packages/spacecat-shared-data-access/test/models/site.test.js b/packages/spacecat-shared-data-access/test/models/site.test.js index 4d1dd5b63..65afd37ae 100644 --- a/packages/spacecat-shared-data-access/test/models/site.test.js +++ b/packages/spacecat-shared-data-access/test/models/site.test.js @@ -14,6 +14,7 @@ import { expect } from 'chai'; import { createSite } from '../../src/models/site.js'; +import { sleep } from '../util.js'; // Constants for testing const validData = { @@ -60,5 +61,31 @@ describe('Site Model Tests', () => { it('throws an error when updating with an empty imsOrgId', () => { expect(() => site.updateImsOrgId('')).to.throw('IMS Org ID must be provided'); }); + + it('sets audits correctly', () => { + const audits = [{ id: 'audit1' }, { id: 'audit2' }]; + site.setAudits(audits); + expect(site.getAudits()).to.deep.equal(audits); + }); + + it('updates updatedAt when base URL is updated', async () => { + const initialUpdatedAt = site.getUpdatedAt(); + + await sleep(10); + + site.updateBaseURL('https://www.newexample.com'); + + expect(site.getUpdatedAt()).to.not.equal(initialUpdatedAt); + }); + + it('updates updatedAt when imsOrgId is updated', async () => { + const initialUpdatedAt = site.getUpdatedAt(); + + await sleep(10); + + site.updateImsOrgId('newOrg123'); + + expect(site.getUpdatedAt()).to.not.equal(initialUpdatedAt); + }); }); }); diff --git a/packages/spacecat-shared-data-access/test/util.js b/packages/spacecat-shared-data-access/test/util.js new file mode 100644 index 000000000..b416000e1 --- /dev/null +++ b/packages/spacecat-shared-data-access/test/util.js @@ -0,0 +1,17 @@ +/* + * Copyright 2023 Adobe. All rights reserved. + * This file is licensed to you under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS + * OF ANY KIND, either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ + +export async function sleep(ms) { + return new Promise((resolve) => { + setTimeout(resolve, ms); + }); +} From ca51663e472d625112ea46fc2b0a30858bcd925d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominique=20J=C3=A4ggi?= Date: Thu, 30 Nov 2023 18:48:35 +0100 Subject: [PATCH 12/28] fix: use array check instead of object --- package-lock.json | 11 +++-------- packages/spacecat-shared-data-access/package.json | 2 +- .../spacecat-shared-data-access/src/models/site.js | 4 ++-- 3 files changed, 6 insertions(+), 11 deletions(-) diff --git a/package-lock.json b/package-lock.json index b5cb53622..7b8081e17 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12265,12 +12265,12 @@ "license": "Apache-2.0", "dependencies": { "@adobe/spacecat-shared-dynamo": "1.1.4", - "@adobe/spacecat-shared-utils": "1.1.0", + "@adobe/spacecat-shared-utils": "1.2.0", + "@aws-sdk/client-dynamodb": "3.454.0", + "@aws-sdk/lib-dynamodb": "3.454.0", "uuid": "9.0.1" }, "devDependencies": { - "@aws-sdk/client-dynamodb": "3.454.0", - "@aws-sdk/lib-dynamodb": "3.454.0", "chai": "4.3.10", "chai-as-promised": "7.1.1", "dynamo-db-local": "6.1.0", @@ -12292,11 +12292,6 @@ "resolved": "https://registry.npmjs.org/@adobe/spacecat-shared-utils/-/spacecat-shared-utils-1.0.1.tgz", "integrity": "sha512-6HiCywan5y9jpbFYkQX87Vg0q5dXoJf2m6MiDvNBEJdREdUxFwE+oYA6Fr86qqd32ECnCeYCFO13HcNRn1RxkQ==" }, - "packages/spacecat-shared-data-access/node_modules/@adobe/spacecat-shared-utils": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@adobe/spacecat-shared-utils/-/spacecat-shared-utils-1.1.0.tgz", - "integrity": "sha512-yq6o47d6IuMApgvlGWdIfUOkqYNwcEm4IzO5WHnrTpWXPK4vvssfAfFtoeki7NQ+A4nFoc7/esqOfX42Q452nA==" - }, "packages/spacecat-shared-data-access/node_modules/uuid": { "version": "9.0.1", "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", diff --git a/packages/spacecat-shared-data-access/package.json b/packages/spacecat-shared-data-access/package.json index 90fea4b06..45e3518c8 100644 --- a/packages/spacecat-shared-data-access/package.json +++ b/packages/spacecat-shared-data-access/package.json @@ -31,7 +31,7 @@ }, "dependencies": { "@adobe/spacecat-shared-dynamo": "1.1.4", - "@adobe/spacecat-shared-utils": "1.1.0", + "@adobe/spacecat-shared-utils": "1.2.0", "@aws-sdk/client-dynamodb": "3.454.0", "@aws-sdk/lib-dynamodb": "3.454.0", "uuid": "9.0.1" diff --git a/packages/spacecat-shared-data-access/src/models/site.js b/packages/spacecat-shared-data-access/src/models/site.js index 14cca4c21..9556db417 100644 --- a/packages/spacecat-shared-data-access/src/models/site.js +++ b/packages/spacecat-shared-data-access/src/models/site.js @@ -10,7 +10,7 @@ * governing permissions and limitations under the License. */ -import { hasText, isObject, isValidUrl } from '@adobe/spacecat-shared-utils'; +import { hasText, isValidUrl } from '@adobe/spacecat-shared-utils'; import { Base } from './base.js'; /** @@ -69,7 +69,7 @@ export const createSite = (data) => { throw new Error('Base URL must be a valid URL'); } - if (!isObject(newState.audits)) { + if (!Array.isArray(newState.audits)) { newState.audits = []; } From d07dc7accc73b332d5262a1d5bcfb36f1cbd450e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominique=20J=C3=A4ggi?= Date: Thu, 30 Nov 2023 18:49:39 +0100 Subject: [PATCH 13/28] chore: typo --- packages/spacecat-shared-data-access/src/dto/audit.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/spacecat-shared-data-access/src/dto/audit.js b/packages/spacecat-shared-data-access/src/dto/audit.js index ba621f439..afcff63d7 100644 --- a/packages/spacecat-shared-data-access/src/dto/audit.js +++ b/packages/spacecat-shared-data-access/src/dto/audit.js @@ -17,7 +17,7 @@ import { createAudit } from '../models/audit.js'; */ export const AuditDto = { /** - * Converts a Audit object into a DynamoDB item. + * Converts an Audit object into a DynamoDB item. * @param {Readonly} audit - Audit object. * @param {boolean} latestAudit - If true, returns the latest audit flavor. * @returns {{siteId, auditedAt, auditResult, auditType, expiresAt, fullAuditRef, SK: string}} @@ -41,7 +41,7 @@ export const AuditDto = { }, /** - * Converts a DynamoDB item into a Audit object. + * Converts a DynamoDB item into an Audit object. * @param {object } dynamoItem - DynamoDB item. * @returns {Readonly} Audit object. */ From 3be0749483bc4584f7ce5298d5f667ec46d092ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominique=20J=C3=A4ggi?= Date: Fri, 1 Dec 2023 07:52:04 +0100 Subject: [PATCH 14/28] fix: bugs and more integration tests --- .../src/audits/accessPatterns.js | 2 +- .../src/dto/audit.js | 13 +- .../src/models/audit.js | 2 + .../src/models/site.js | 30 ++- .../test-it/db.test.js | 232 +++++++++++++++++- .../test/models/site.test.js | 10 +- .../test/sites/index.test.js | 2 +- 7 files changed, 276 insertions(+), 15 deletions(-) diff --git a/packages/spacecat-shared-data-access/src/audits/accessPatterns.js b/packages/spacecat-shared-data-access/src/audits/accessPatterns.js index a3b0ac77d..aace72881 100644 --- a/packages/spacecat-shared-data-access/src/audits/accessPatterns.js +++ b/packages/spacecat-shared-data-access/src/audits/accessPatterns.js @@ -67,7 +67,7 @@ export const getAuditForSite = async ( KeyConditionExpression: 'siteId = :siteId AND SK = :sk', ExpressionAttributeValues: { ':siteId': siteId, - ':sk': `${auditType}#${auditedAt}}`, + ':sk': `${auditType}#${auditedAt}`, }, Limit: 1, }); diff --git a/packages/spacecat-shared-data-access/src/dto/audit.js b/packages/spacecat-shared-data-access/src/dto/audit.js index afcff63d7..156fc5645 100644 --- a/packages/spacecat-shared-data-access/src/dto/audit.js +++ b/packages/spacecat-shared-data-access/src/dto/audit.js @@ -12,6 +12,15 @@ import { createAudit } from '../models/audit.js'; +function parseEpochToDate(epochInSeconds) { + const milliseconds = epochInSeconds * 1000; + return new Date(milliseconds); +} + +function convertDateToEpochSeconds(date) { + return Math.floor(date.getTime() / 1000); +} + /** * Data transfer object for Audit. */ @@ -33,7 +42,7 @@ export const AuditDto = { auditedAt: audit.getAuditedAt(), auditResult: audit.getAuditResult(), auditType: audit.getAuditType(), - expiresAt: audit.getExpiresAt(), + expiresAt: convertDateToEpochSeconds(audit.getExpiresAt()), fullAuditRef: audit.getFullAuditRef(), SK: `${audit.getAuditType()}#${audit.getAuditedAt()}`, ...latestAuditProps, @@ -51,7 +60,7 @@ export const AuditDto = { auditedAt: dynamoItem.auditedAt, auditResult: dynamoItem.auditResult, auditType: dynamoItem.auditType, - expiresAt: dynamoItem.expiresAt, + expiresAt: parseEpochToDate(dynamoItem.expiresAt), fullAuditRef: dynamoItem.fullAuditRef, }; diff --git a/packages/spacecat-shared-data-access/src/models/audit.js b/packages/spacecat-shared-data-access/src/models/audit.js index 82cd68dc3..1f4b9ef81 100644 --- a/packages/spacecat-shared-data-access/src/models/audit.js +++ b/packages/spacecat-shared-data-access/src/models/audit.js @@ -13,11 +13,13 @@ import { hasText, isIsoDate, isObject } from '@adobe/spacecat-shared-utils'; import { Base } from './base.js'; +export const AUDIT_TYPE_CWV = 'cwv'; export const AUDIT_TYPE_LHS = 'lhs'; const EXPIRES_IN_DAYS = 30; const AUDIT_TYPE_PROPERTIES = { + [AUDIT_TYPE_CWV]: [], [AUDIT_TYPE_LHS]: ['performance', 'seo', 'accessibility', 'best-practices'], }; diff --git a/packages/spacecat-shared-data-access/src/models/site.js b/packages/spacecat-shared-data-access/src/models/site.js index 9556db417..94fe8cc35 100644 --- a/packages/spacecat-shared-data-access/src/models/site.js +++ b/packages/spacecat-shared-data-access/src/models/site.js @@ -26,7 +26,33 @@ const Site = (data = {}) => { self.getBaseURL = () => self.state.baseURL; self.getImsOrgId = () => self.state.imsOrgId; - self.updateBaseURL = (baseURL) => { + // TODO: updating the baseURL is not supported yet, it will require a transact write + // on dynamodb (put then delete) since baseURL is part of the primary key, something like: + // const updateSiteBaseURL = async (oldBaseURL, updatedSiteData) => { + // const params = { + // TransactItems: [ + // { + // Put: { + // TableName: 'YourSiteTableName', + // Item: updatedSiteData, + // }, + // }, + // { + // Delete: { + // TableName: 'YourSiteTableName', + // Key: { + // baseURL: oldBaseURL, + // }, + // }, + // }, + // ], + // }; + // + // await dynamoDbClient.transactWrite(params).promise(); + // + // return createSite(updatedSiteData); + // }; + /* self.updateBaseURL = (baseURL) => { if (!isValidUrl(baseURL)) { throw new Error('Base URL must be a valid URL'); } @@ -35,7 +61,7 @@ const Site = (data = {}) => { self.touch(); return self; - }; + }; */ self.updateImsOrgId = (imsOrgId) => { if (!hasText(imsOrgId)) { diff --git a/packages/spacecat-shared-data-access/test-it/db.test.js b/packages/spacecat-shared-data-access/test-it/db.test.js index 30258dff2..0718a234b 100644 --- a/packages/spacecat-shared-data-access/test-it/db.test.js +++ b/packages/spacecat-shared-data-access/test-it/db.test.js @@ -15,7 +15,7 @@ import { expect } from 'chai'; import dynamoDbLocal from 'dynamo-db-local'; -import { isValidUrl } from '@adobe/spacecat-shared-utils'; +import { isIsoDate, isValidUrl } from '@adobe/spacecat-shared-utils'; import { sleep } from '../test/util.js'; import { createDataAccess } from '../src/index.js'; @@ -24,20 +24,33 @@ import { AUDIT_TYPE_LHS } from '../src/models/audit.js'; import generateSampleData from './generateSampleData.js'; function checkSite(site) { + expect(site).to.be.an('object'); expect(site.getId()).to.be.a('string'); expect(site.getBaseURL()).to.be.a('string'); expect(site.getImsOrgId()).to.be.a('string'); - expect(site.getCreatedAt()).to.be.a('string'); - expect(site.getUpdatedAt()).to.be.a('string'); + expect(isIsoDate(site.getCreatedAt())).to.be.true; + expect(isIsoDate(site.getUpdatedAt())).to.be.true; expect(site.getAudits()).to.be.an('array'); } +function checkAudit(audit) { + expect(audit).to.be.an('object'); + expect(audit.getId()).to.be.a('string'); + expect(audit.getSiteId()).to.be.a('string'); + expect(audit.getAuditType()).to.be.a('string'); + expect(isIsoDate(audit.getAuditedAt())).to.be.true; + expect(audit.getExpiresAt()).to.be.a('date'); + expect(audit.getAuditResult()).to.be.an('object'); + expect(audit.getScores()).to.be.an('object'); + expect(audit.getFullAuditRef()).to.be.a('string'); +} + describe('DynamoDB Integration Test', async () => { let dynamoDbLocalProcess; let dataAccess; const NUMBER_OF_SITES = 10; - const NUMBER_OF_AUDITS_PER_SITE = 3; + const NUMBER_OF_AUDITS_PER_TYPE_AND_SITE = 3; before(async function () { this.timeout(3000); @@ -46,9 +59,9 @@ describe('DynamoDB Integration Test', async () => { dynamoDbLocalProcess = dynamoDbLocal.spawn({ port: 8000, sharedDb: true }); - await sleep(500); + await sleep(700); // give db time to start up - await generateSampleData(NUMBER_OF_SITES, NUMBER_OF_AUDITS_PER_SITE); + await generateSampleData(NUMBER_OF_SITES, NUMBER_OF_AUDITS_PER_TYPE_AND_SITE); dataAccess = createDataAccess(console); }); @@ -104,4 +117,211 @@ describe('DynamoDB Integration Test', async () => { checkSite(site); }); + + it('adds a new site', async () => { + const newSiteData = { + baseURL: 'https://newexample.com', + imsOrgId: 'newOrg123', + audits: [], + }; + + const addedSite = await dataAccess.addSite(newSiteData); + + expect(addedSite).to.be.an('object'); + + const newSite = await dataAccess.getSiteByBaseURL(newSiteData.baseURL); + + checkSite(newSite); + + expect(newSite.getId()).to.to.be.a('string'); + expect(newSite.getBaseURL()).to.equal(newSiteData.baseURL); + expect(newSite.getImsOrgId()).to.equal(newSiteData.imsOrgId); + expect(newSite.getAudits()).to.be.an('array').that.is.empty; + }); + + it('updates an existing site', async () => { + const siteToUpdate = await dataAccess.getSiteByBaseURL('https://example1.com'); + const originalUpdatedAt = siteToUpdate.getUpdatedAt(); + const newImsOrgId = 'updatedOrg123'; + + await sleep(10); // Make sure updatedAt is different + + siteToUpdate.updateImsOrgId(newImsOrgId); + + const updatedSite = await dataAccess.updateSite(siteToUpdate); + + expect(updatedSite.getImsOrgId()).to.equal(newImsOrgId); + expect(updatedSite.getUpdatedAt()).to.not.equal(originalUpdatedAt); + }); + + it('retrieves all audits for a site', async () => { + const site = await dataAccess.getSiteByBaseURL('https://example1.com'); + const siteId = site.getId(); + const audits = await dataAccess.getAuditsForSite(siteId); + + expect(audits).to.be.an('array').that.has.lengthOf(NUMBER_OF_AUDITS_PER_TYPE_AND_SITE * 2); + + audits.forEach((audit) => { + checkAudit(audit); + expect(audit.getSiteId()).to.equal(siteId); + }); + }); + + it('retrieves audits of a specific type for a site', async () => { + const site = await dataAccess.getSiteByBaseURL('https://example1.com'); + const siteId = site.getId(); + const auditType = AUDIT_TYPE_LHS; + const audits = await dataAccess.getAuditsForSite(siteId, auditType); + + expect(audits).to.be.an('array').that.has.lengthOf(NUMBER_OF_AUDITS_PER_TYPE_AND_SITE); + + audits.forEach((audit) => { + checkAudit(audit); + expect(audit.getSiteId()).to.equal(siteId); + expect(audit.getAuditType()).to.equal(auditType); + }); + }); + + it('retrieves a specific audit for a site', async () => { + const site = await dataAccess.getSiteByBaseURL('https://example1.com'); + const siteId = site.getId(); + const auditType = AUDIT_TYPE_LHS; + const audits = await dataAccess.getAuditsForSite(site.getId(), auditType); + const auditedAt = audits[0].getAuditedAt(); + + const audit = await dataAccess.getAuditForSite(siteId, auditType, auditedAt); + + checkAudit(audit); + + expect(audit.getSiteId()).to.equal(siteId); + expect(audit.getAuditType()).to.equal(auditType); + expect(audit.getAuditedAt()).to.equal(auditedAt); + }); + + it('returns null for non-existing audit', async () => { + const site = await dataAccess.getSiteByBaseURL('https://example1.com'); + const siteId = site.getId(); + const auditType = 'non-existing-type'; + const auditedAt = '2023-01-01T00:00:00Z'; + + const audit = await dataAccess.getAuditForSite(siteId, auditType, auditedAt); + + expect(audit).to.be.null; + }); + + it('retrieves the latest audits of a specific type', async () => { + const audits = await dataAccess.getLatestAudits(AUDIT_TYPE_LHS, true); + + // Every tenth site will not have any audits + expect(audits).to.be.an('array').that.has.lengthOf(NUMBER_OF_SITES - 1); + + audits.forEach((audit) => { + checkAudit(audit); + expect(audit.getAuditType()).to.equal(AUDIT_TYPE_LHS); + }); + + // verify the sorting order + let lastScoresString = ''; + audits.forEach((audit) => { + const currentScoresString = `${AUDIT_TYPE_LHS}#${Object.keys(audit.getScores()).join('#')}`; + expect(currentScoresString.localeCompare(lastScoresString)).to.be.at.least(0); + lastScoresString = currentScoresString; + }); + }); + + it('retrieves the latest audits in descending order', async () => { + const audits = await dataAccess.getLatestAudits(AUDIT_TYPE_LHS, false); + + expect(audits).to.be.an('array').that.has.lengthOf(NUMBER_OF_SITES - 1); + + // verify the sorting order is descending + // assuming 'z' will be lexicographically after any realistic score string + let lastScoresString = 'z'; + audits.forEach((audit) => { + const currentScoresString = `${AUDIT_TYPE_LHS}#${Object.keys(audit.getScores()).join('#')}`; + expect(currentScoresString.localeCompare(lastScoresString)).to.be.at.most(0); + lastScoresString = currentScoresString; + }); + }); + + it('retrieves the latest audit for a specific site and audit type', async () => { + const site = await dataAccess.getSiteByBaseURL('https://example1.com'); + const siteId = site.getId(); + + const latestAudit = await dataAccess.getLatestAuditForSite(siteId, AUDIT_TYPE_LHS); + + checkAudit(latestAudit); + expect(latestAudit.getSiteId()).to.equal(siteId); + expect(latestAudit.getAuditType()).to.equal(AUDIT_TYPE_LHS); + + const allAudits = await dataAccess.getAuditsForSite(siteId, AUDIT_TYPE_LHS); + const mostRecentAudit = allAudits.reduce((latest, current) => ( + new Date(latest.getAuditedAt()) > new Date(current.getAuditedAt()) ? latest : current + )); + + expect(latestAudit.getAuditedAt()).to.equal(mostRecentAudit.getAuditedAt()); + }); + + it('returns null for a site with no audits of the specified type', async () => { + const site = await dataAccess.getSiteByBaseURL('https://example1.com'); + const siteId = site.getId(); + const auditType = 'non-existing-type'; + + const latestAudit = await dataAccess.getLatestAuditForSite(siteId, auditType); + + expect(latestAudit).to.be.null; + }); + + it('successfully adds a new audit', async () => { + const auditData = { + siteId: 'https://example1.com', + auditType: AUDIT_TYPE_LHS, + auditedAt: new Date().toISOString(), + fullAuditRef: 's3://ref', + auditResult: { + performance: 0, + seo: 0, + accessibility: 0, + 'best-practices': 0, + }, + }; + + const newAudit = await dataAccess.addAudit(auditData); + + checkAudit(newAudit); + expect(newAudit.getSiteId()).to.equal(auditData.siteId); + expect(newAudit.getAuditType()).to.equal(auditData.auditType); + expect(newAudit.getAuditedAt()).to.equal(auditData.auditedAt); + + // Retrieve the latest audit for the site from the latest_audits table + const latestAudit = await dataAccess.getLatestAuditForSite( + auditData.siteId, + auditData.auditType, + ); + + checkAudit(latestAudit); + expect(latestAudit.getSiteId()).to.equal(auditData.siteId); + expect(latestAudit.getAuditType()).to.equal(auditData.auditType); + expect(latestAudit.getAuditedAt()).to.equal(auditData.auditedAt); + }); + + it('throws an error when adding a duplicate audit', async () => { + const auditData = { + siteId: 'https://example1.com', + auditType: AUDIT_TYPE_LHS, + auditedAt: new Date().toISOString(), + fullAuditRef: 's3://ref', + auditResult: { + performance: 0, + seo: 0, + accessibility: 0, + 'best-practices': 0, + }, + }; + + await dataAccess.addAudit(auditData); + + // Try to add the same audit again + await expect(dataAccess.addAudit(auditData)).to.eventually.be.rejectedWith('Audit already exists'); + }); }); diff --git a/packages/spacecat-shared-data-access/test/models/site.test.js b/packages/spacecat-shared-data-access/test/models/site.test.js index 65afd37ae..5489f1ee6 100644 --- a/packages/spacecat-shared-data-access/test/models/site.test.js +++ b/packages/spacecat-shared-data-access/test/models/site.test.js @@ -42,7 +42,8 @@ describe('Site Model Tests', () => { site = createSite(validData); }); - it('updates baseURL correctly', () => { + // see TODO in src/models/site.js + /* it('updates baseURL correctly', () => { const newURL = 'https://www.newexample.com'; site.updateBaseURL(newURL); expect(site.getBaseURL()).to.equal(newURL); @@ -51,6 +52,7 @@ describe('Site Model Tests', () => { it('throws an error when updating with an invalid baseURL', () => { expect(() => site.updateBaseURL('invalid-url')).to.throw('Base URL must be a valid URL'); }); + */ it('updates imsOrgId correctly', () => { const newImsOrgId = 'newOrg123'; @@ -68,6 +70,8 @@ describe('Site Model Tests', () => { expect(site.getAudits()).to.deep.equal(audits); }); + // see TODO in src/models/site.js + /* it('updates updatedAt when base URL is updated', async () => { const initialUpdatedAt = site.getUpdatedAt(); @@ -76,12 +80,12 @@ describe('Site Model Tests', () => { site.updateBaseURL('https://www.newexample.com'); expect(site.getUpdatedAt()).to.not.equal(initialUpdatedAt); - }); + }); */ it('updates updatedAt when imsOrgId is updated', async () => { const initialUpdatedAt = site.getUpdatedAt(); - await sleep(10); + await sleep(20); site.updateImsOrgId('newOrg123'); diff --git a/packages/spacecat-shared-data-access/test/sites/index.test.js b/packages/spacecat-shared-data-access/test/sites/index.test.js index 51376bb80..ac71dc507 100644 --- a/packages/spacecat-shared-data-access/test/sites/index.test.js +++ b/packages/spacecat-shared-data-access/test/sites/index.test.js @@ -304,7 +304,7 @@ describe('Site Access Pattern Tests', () => { mockDynamoClient.query.returns(Promise.resolve([siteData])); const site = await exportedFunctions.getSiteByBaseURL(siteData.baseURL); - site.updateBaseURL('https://newsite.com'); + // site.updateBaseURL('https://newsite.com'); site.updateImsOrgId('newOrg123'); const result = await exportedFunctions.updateSite(site); From 32ec0891ae201bcb348e2019c99477738cd36b5d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominique=20J=C3=A4ggi?= Date: Fri, 1 Dec 2023 07:59:56 +0100 Subject: [PATCH 15/28] fix: integration test rejectedWith --- packages/spacecat-shared-data-access/package.json | 7 +++---- .../spacecat-shared-data-access/test-it/db.test.js | 11 +++++++---- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/packages/spacecat-shared-data-access/package.json b/packages/spacecat-shared-data-access/package.json index 45e3518c8..47f543f9d 100644 --- a/packages/spacecat-shared-data-access/package.json +++ b/packages/spacecat-shared-data-access/package.json @@ -6,15 +6,14 @@ "main": "src/index.js", "types": "src/index.d.ts", "scripts": { - "test:it": "mocha test-it/", - "test": "c8 mocha test/", + "test:it": "mocha --spec \"test-it/**/*.test.js\"", + "test": "c8 mocha --spec \"test/**/*.test.js\"", "lint": "eslint .", "clean": "rm -rf package-lock.json node_modules" }, "mocha": { "reporter": "mocha-multi-reporters", - "reporter-options": "configFile=.mocha-multi.json", - "spec": "test/**/*.test.js" + "reporter-options": "configFile=.mocha-multi.json" }, "repository": { "type": "git", diff --git a/packages/spacecat-shared-data-access/test-it/db.test.js b/packages/spacecat-shared-data-access/test-it/db.test.js index 0718a234b..be7df18a1 100644 --- a/packages/spacecat-shared-data-access/test-it/db.test.js +++ b/packages/spacecat-shared-data-access/test-it/db.test.js @@ -12,17 +12,20 @@ /* eslint-env mocha */ -import { expect } from 'chai'; - +import chai from 'chai'; +import chaiAsPromised from 'chai-as-promised'; import dynamoDbLocal from 'dynamo-db-local'; -import { isIsoDate, isValidUrl } from '@adobe/spacecat-shared-utils'; +import { isIsoDate, isValidUrl } from '@adobe/spacecat-shared-utils'; import { sleep } from '../test/util.js'; import { createDataAccess } from '../src/index.js'; import { AUDIT_TYPE_LHS } from '../src/models/audit.js'; import generateSampleData from './generateSampleData.js'; +const { expect } = chai; +chai.use(chaiAsPromised); + function checkSite(site) { expect(site).to.be.an('object'); expect(site.getId()).to.be.a('string'); @@ -322,6 +325,6 @@ describe('DynamoDB Integration Test', async () => { await dataAccess.addAudit(auditData); // Try to add the same audit again - await expect(dataAccess.addAudit(auditData)).to.eventually.be.rejectedWith('Audit already exists'); + await expect(dataAccess.addAudit(auditData)).to.be.rejectedWith('Audit already exists'); }); }); From 3fb1b02662071662e0735ce63463c013cdad179b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominique=20J=C3=A4ggi?= Date: Fri, 1 Dec 2023 08:14:17 +0100 Subject: [PATCH 16/28] chore: run integration tests in CI --- .circleci/config.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 99fb81f52..495f6fe64 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -34,6 +34,9 @@ jobs: - run: name: Running tests and getting code coverage command: cat /tmp/tests-to-run | xargs -I % npm run test -w % + - run: + name: Running data access integration tests + command: npm run test:it -w packages/spacecat-shared-data-access - codecov/upload - run: name: Copy test results From 38e1748d5bffc45646516cc338bd493ea7a99484 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominique=20J=C3=A4ggi?= Date: Fri, 1 Dec 2023 08:17:34 +0100 Subject: [PATCH 17/28] fix: add java to ci image --- .circleci/config.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 495f6fe64..acf550569 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,8 +1,8 @@ version: 2.1 executors: - node18: + node-java: docker: - - image: cimg/node:18.18 + - image: cimg/node:18.18-browsers orbs: codecov: codecov/codecov@3.3.0 @@ -20,7 +20,7 @@ commands: jobs: build: - executor: node18 + executor: node-java parallelism: 15 steps: From 9545fc4eb053428758150ba8385fb274d59bc3f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominique=20J=C3=A4ggi?= Date: Fri, 1 Dec 2023 08:19:12 +0100 Subject: [PATCH 18/28] fix: adjust dynamo launch for CI env --- packages/spacecat-shared-data-access/test-it/db.test.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/spacecat-shared-data-access/test-it/db.test.js b/packages/spacecat-shared-data-access/test-it/db.test.js index be7df18a1..febd722dd 100644 --- a/packages/spacecat-shared-data-access/test-it/db.test.js +++ b/packages/spacecat-shared-data-access/test-it/db.test.js @@ -56,13 +56,13 @@ describe('DynamoDB Integration Test', async () => { const NUMBER_OF_AUDITS_PER_TYPE_AND_SITE = 3; before(async function () { - this.timeout(3000); + this.timeout(20000); process.env.AWS_REGION = 'local'; dynamoDbLocalProcess = dynamoDbLocal.spawn({ port: 8000, sharedDb: true }); - await sleep(700); // give db time to start up + await sleep(1000); // give db time to start up await generateSampleData(NUMBER_OF_SITES, NUMBER_OF_AUDITS_PER_TYPE_AND_SITE); From a70fd89c668311571714232cdeeee45d8608281b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominique=20J=C3=A4ggi?= Date: Fri, 1 Dec 2023 08:23:03 +0100 Subject: [PATCH 19/28] fix: provide dummy creds for it-test --- packages/spacecat-shared-data-access/test-it/db.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/spacecat-shared-data-access/test-it/db.js b/packages/spacecat-shared-data-access/test-it/db.js index 7bff40e77..391712498 100644 --- a/packages/spacecat-shared-data-access/test-it/db.js +++ b/packages/spacecat-shared-data-access/test-it/db.js @@ -16,6 +16,10 @@ import { DynamoDBDocument } from '@aws-sdk/lib-dynamodb'; const dbClient = new DynamoDB({ endpoint: 'http://127.0.0.1:8000', region: 'local', + credentials: { + accessKeyId: 'dummy', + secretAccessKey: 'dummy', + }, }); const docClient = DynamoDBDocument.from(dbClient); From 4f3aae6af8d10520e00ecf4fcd2745e47275325d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominique=20J=C3=A4ggi?= Date: Fri, 1 Dec 2023 08:27:44 +0100 Subject: [PATCH 20/28] fix: set dummy creds in env --- packages/spacecat-shared-data-access/test-it/db.test.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/spacecat-shared-data-access/test-it/db.test.js b/packages/spacecat-shared-data-access/test-it/db.test.js index febd722dd..63de90031 100644 --- a/packages/spacecat-shared-data-access/test-it/db.test.js +++ b/packages/spacecat-shared-data-access/test-it/db.test.js @@ -59,6 +59,9 @@ describe('DynamoDB Integration Test', async () => { this.timeout(20000); process.env.AWS_REGION = 'local'; + process.env.AWS_DEFAULT_REGION = 'local'; + process.env.AWS_ACCESS_KEY_ID = 'dummy'; + process.env.AWS_SECRET_ACCESS_KEY = 'dummy'; dynamoDbLocalProcess = dynamoDbLocal.spawn({ port: 8000, sharedDb: true }); From 67c2ec2cc6d95604723e76dadf52efc880a9f79c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominique=20J=C3=A4ggi?= Date: Fri, 1 Dec 2023 08:32:42 +0100 Subject: [PATCH 21/28] fix: only run IT tests if needed --- .circleci/config.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index acf550569..f8a73a67b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -35,8 +35,14 @@ jobs: name: Running tests and getting code coverage command: cat /tmp/tests-to-run | xargs -I % npm run test -w % - run: - name: Running data access integration tests - command: npm run test:it -w packages/spacecat-shared-data-access + name: Running integration tests + command: | + if grep -q "packages/spacecat-shared-data-access" /tmp/tests-to-run; then + echo "Running integration tests for spacecat-shared-data-access" + npm run test:it -w packages/spacecat-shared-data-access + else + echo "Skipping integration tests" + fi - codecov/upload - run: name: Copy test results From ab64662baec5876f4e7789150dd8fdfabe23d13c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominique=20J=C3=A4ggi?= Date: Fri, 1 Dec 2023 08:41:14 +0100 Subject: [PATCH 22/28] fix: directory structure --- packages/spacecat-shared-data-access/src/index.js | 4 ++-- .../src/{ => service}/audits/accessPatterns.js | 4 ++-- .../src/{ => service}/audits/index.js | 0 .../src/{ => service}/sites/accessPatterns.js | 4 ++-- .../src/{ => service}/sites/index.js | 0 .../test/{ => service}/audits/index.test.js | 2 +- .../test/{ => service}/sites/index.test.js | 4 ++-- 7 files changed, 9 insertions(+), 9 deletions(-) rename packages/spacecat-shared-data-access/src/{ => service}/audits/accessPatterns.js (98%) rename packages/spacecat-shared-data-access/src/{ => service}/audits/index.js (100%) rename packages/spacecat-shared-data-access/src/{ => service}/sites/accessPatterns.js (98%) rename packages/spacecat-shared-data-access/src/{ => service}/sites/index.js (100%) rename packages/spacecat-shared-data-access/test/{ => service}/audits/index.test.js (98%) rename packages/spacecat-shared-data-access/test/{ => service}/sites/index.test.js (98%) diff --git a/packages/spacecat-shared-data-access/src/index.js b/packages/spacecat-shared-data-access/src/index.js index 6f0b9bddf..2b6006e82 100644 --- a/packages/spacecat-shared-data-access/src/index.js +++ b/packages/spacecat-shared-data-access/src/index.js @@ -11,8 +11,8 @@ */ import { createClient } from '@adobe/spacecat-shared-dynamo'; -import { auditFunctions } from './audits/index.js'; -import { siteFunctions } from './sites/index.js'; +import { auditFunctions } from './service/audits/index.js'; +import { siteFunctions } from './service/sites/index.js'; /** * Creates a data access object. diff --git a/packages/spacecat-shared-data-access/src/audits/accessPatterns.js b/packages/spacecat-shared-data-access/src/service/audits/accessPatterns.js similarity index 98% rename from packages/spacecat-shared-data-access/src/audits/accessPatterns.js rename to packages/spacecat-shared-data-access/src/service/audits/accessPatterns.js index aace72881..cbda82f1e 100644 --- a/packages/spacecat-shared-data-access/src/audits/accessPatterns.js +++ b/packages/spacecat-shared-data-access/src/service/audits/accessPatterns.js @@ -12,8 +12,8 @@ import { isObject } from '@adobe/spacecat-shared-utils'; -import { AuditDto } from '../dto/audit.js'; -import { createAudit } from '../models/audit.js'; +import { AuditDto } from '../../dto/audit.js'; +import { createAudit } from '../../models/audit.js'; /** * Retrieves audits for a specified site. If an audit type is provided, diff --git a/packages/spacecat-shared-data-access/src/audits/index.js b/packages/spacecat-shared-data-access/src/service/audits/index.js similarity index 100% rename from packages/spacecat-shared-data-access/src/audits/index.js rename to packages/spacecat-shared-data-access/src/service/audits/index.js diff --git a/packages/spacecat-shared-data-access/src/sites/accessPatterns.js b/packages/spacecat-shared-data-access/src/service/sites/accessPatterns.js similarity index 98% rename from packages/spacecat-shared-data-access/src/sites/accessPatterns.js rename to packages/spacecat-shared-data-access/src/service/sites/accessPatterns.js index 7881d69de..4aab3a33d 100644 --- a/packages/spacecat-shared-data-access/src/sites/accessPatterns.js +++ b/packages/spacecat-shared-data-access/src/service/sites/accessPatterns.js @@ -18,8 +18,8 @@ import { getLatestAudits, } from '../audits/accessPatterns.js'; -import { createSite } from '../models/site.js'; -import { SiteDto } from '../dto/site.js'; +import { createSite } from '../../models/site.js'; +import { SiteDto } from '../../dto/site.js'; const INDEX_NAME_ALL_SITES = 'all_sites'; const PK_ALL_SITES = 'ALL_SITES'; diff --git a/packages/spacecat-shared-data-access/src/sites/index.js b/packages/spacecat-shared-data-access/src/service/sites/index.js similarity index 100% rename from packages/spacecat-shared-data-access/src/sites/index.js rename to packages/spacecat-shared-data-access/src/service/sites/index.js diff --git a/packages/spacecat-shared-data-access/test/audits/index.test.js b/packages/spacecat-shared-data-access/test/service/audits/index.test.js similarity index 98% rename from packages/spacecat-shared-data-access/test/audits/index.test.js rename to packages/spacecat-shared-data-access/test/service/audits/index.test.js index ce3cf4c3a..2d127b046 100644 --- a/packages/spacecat-shared-data-access/test/audits/index.test.js +++ b/packages/spacecat-shared-data-access/test/service/audits/index.test.js @@ -15,7 +15,7 @@ import { expect } from 'chai'; import sinon from 'sinon'; -import { auditFunctions } from '../../src/audits/index.js'; +import { auditFunctions } from '../../../src/service/audits/index.js'; describe('Audit Access Pattern Tests', () => { describe('Audit Functions Export Tests', () => { diff --git a/packages/spacecat-shared-data-access/test/sites/index.test.js b/packages/spacecat-shared-data-access/test/service/sites/index.test.js similarity index 98% rename from packages/spacecat-shared-data-access/test/sites/index.test.js rename to packages/spacecat-shared-data-access/test/service/sites/index.test.js index ac71dc507..3dcf23463 100644 --- a/packages/spacecat-shared-data-access/test/sites/index.test.js +++ b/packages/spacecat-shared-data-access/test/service/sites/index.test.js @@ -16,8 +16,8 @@ import chai from 'chai'; import chaiAsPromised from 'chai-as-promised'; import sinon from 'sinon'; -import { siteFunctions } from '../../src/sites/index.js'; -import { createSite } from '../../src/models/site.js'; +import { siteFunctions } from '../../../src/service/sites/index.js'; +import { createSite } from '../../../src/models/site.js'; chai.use(chaiAsPromised); From 5c2499859b250fce723fc03b5fc81deb5fd3648c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominique=20J=C3=A4ggi?= Date: Fri, 1 Dec 2023 09:38:40 +0100 Subject: [PATCH 23/28] feat: add removeSite pattern --- .../src/index.d.ts | 6 ++ .../src/service/audits/accessPatterns.js | 73 +++++++++++++++++-- .../src/service/audits/index.js | 6 ++ .../src/service/sites/accessPatterns.js | 21 +++++- .../src/service/sites/index.js | 3 +- .../test/index.test.js | 2 + .../test/service/audits/index.test.js | 61 +++++++++++++++- .../test/service/sites/index.test.js | 32 ++++++++ 8 files changed, 192 insertions(+), 12 deletions(-) diff --git a/packages/spacecat-shared-data-access/src/index.d.ts b/packages/spacecat-shared-data-access/src/index.d.ts index 3aa03b507..b2aef3069 100644 --- a/packages/spacecat-shared-data-access/src/index.d.ts +++ b/packages/spacecat-shared-data-access/src/index.d.ts @@ -45,6 +45,9 @@ export interface DataAccess { auditType: string, ascending?: boolean, ) => Promise; + getLatestAuditsForSite: ( + siteId: string, + ) => Promise; getSites: () => Promise; getSitesToAudit: () => Promise; getSitesWithLatestAudit: ( @@ -73,6 +76,9 @@ export interface DataAccess { updateSite: ( site: Site, ) => Promise; + removeSite: ( + siteId: string, + ) => Promise; } export function createDataAccess( diff --git a/packages/spacecat-shared-data-access/src/service/audits/accessPatterns.js b/packages/spacecat-shared-data-access/src/service/audits/accessPatterns.js index cbda82f1e..e4e4439ca 100644 --- a/packages/spacecat-shared-data-access/src/service/audits/accessPatterns.js +++ b/packages/spacecat-shared-data-access/src/service/audits/accessPatterns.js @@ -15,6 +15,11 @@ import { isObject } from '@adobe/spacecat-shared-utils'; import { AuditDto } from '../../dto/audit.js'; import { createAudit } from '../../models/audit.js'; +const TABLE_NAME_AUDITS = 'audits'; +const TABLE_NAME_LATEST_AUDITS = 'latest_audits'; +const INDEX_NAME_ALL_LATEST_AUDIT_SCORES = 'all_latest_audit_scores'; +const PK_ALL_LATEST_AUDITS = 'ALL_LATEST_AUDITS'; + /** * Retrieves audits for a specified site. If an audit type is provided, * it returns only audits of that type. @@ -26,9 +31,8 @@ import { createAudit } from '../../models/audit.js'; * @returns {Promise} A promise that resolves to an array of audits for the specified site. */ export const getAuditsForSite = async (dynamoClient, log, siteId, auditType) => { - // Base query parameters const queryParams = { - TableName: 'audits', + TableName: TABLE_NAME_AUDITS, KeyConditionExpression: 'siteId = :siteId', ExpressionAttributeValues: { ':siteId': siteId, @@ -63,7 +67,7 @@ export const getAuditForSite = async ( auditedAt, ) => { const audit = await dynamoClient.query({ - TableName: 'audits', + TableName: TABLE_NAME_AUDITS, KeyConditionExpression: 'siteId = :siteId AND SK = :sk', ExpressionAttributeValues: { ':siteId': siteId, @@ -93,11 +97,11 @@ export const getLatestAudits = async ( ascending = true, ) => { const dynamoItems = await dynamoClient.query({ - TableName: 'latest_audits', - IndexName: 'all_latest_audit_scores', + TableName: TABLE_NAME_LATEST_AUDITS, + IndexName: INDEX_NAME_ALL_LATEST_AUDIT_SCORES, KeyConditionExpression: 'GSI1PK = :gsi1pk AND begins_with(GSI1SK, :auditType)', ExpressionAttributeValues: { - ':gsi1pk': 'ALL_LATEST_AUDITS', + ':gsi1pk': PK_ALL_LATEST_AUDITS, ':auditType': `${auditType}#`, }, ScanIndexForward: ascending, // Sorts ascending if true, descending if false @@ -106,6 +110,27 @@ export const getLatestAudits = async ( return dynamoItems.map((item) => AuditDto.fromDynamoItem(item)); }; +/** + * Retrieves latest audits for a specified site. + * + * @param {DynamoDbClient} dynamoClient - The DynamoDB client. + * @param {Logger} log - The logger. + * @param {string} siteId - The ID of the site for which audits are being retrieved. + * @returns {Promise} A promise that resolves to an array of latest audits + * for the specified site. + */ +export const getLatestAuditsForSite = async (dynamoClient, log, siteId) => { + const queryParams = { + TableName: TABLE_NAME_LATEST_AUDITS, + KeyConditionExpression: 'siteId = :siteId', + ExpressionAttributeValues: { ':siteId': siteId }, + }; + + const dynamoItems = await dynamoClient.query(queryParams); + + return dynamoItems.map((item) => AuditDto.fromDynamoItem(item)); +}; + /** * Retrieves the latest audit for a specified site and audit type. * @@ -123,7 +148,7 @@ export const getLatestAuditForSite = async ( auditType, ) => { const latestAudit = await dynamoClient.query({ - TableName: 'latest_audits', + TableName: TABLE_NAME_LATEST_AUDITS, KeyConditionExpression: 'siteId = :siteId AND begins_with(SK, :auditType)', ExpressionAttributeValues: { ':siteId': siteId, @@ -163,3 +188,37 @@ export const addAudit = async (dynamoClient, log, auditData) => { return audit; }; + +async function removeAudits(dynamoClient, audits) { + // TODO: use batch-remove (needs dynamo client update) + const removeAuditPromises = audits.map((audit) => dynamoClient.removeItem({ + TableName: TABLE_NAME_AUDITS, + Key: { + siteId: audit.getSiteId(), + auditedAt: audit.getAuditedAt(), + }, + })); + + await Promise.all(removeAuditPromises); +} + +/** + * Removes all audits for a specified site and the latest audit entry. + * + * @param {DynamoDbClient} dynamoClient - The DynamoDB client. + * @param {Logger} log - The logger. + * @param {string} siteId - The ID of the site for which audits are being removed. + * @returns {Promise} + */ +export const removeAuditsForSite = async (dynamoClient, log, siteId) => { + try { + const audits = await getAuditsForSite(dynamoClient, log, siteId); + const latestAudits = await getLatestAuditsForSite(dynamoClient, log, siteId); + + await removeAudits(dynamoClient, audits); + await removeAudits(dynamoClient, latestAudits); + } catch (error) { + log.error(`Error removing audits for site ${siteId}: ${error.message}`); + throw error; + } +}; diff --git a/packages/spacecat-shared-data-access/src/service/audits/index.js b/packages/spacecat-shared-data-access/src/service/audits/index.js index 87645dde7..f21e86b27 100644 --- a/packages/spacecat-shared-data-access/src/service/audits/index.js +++ b/packages/spacecat-shared-data-access/src/service/audits/index.js @@ -15,6 +15,7 @@ import { getAuditsForSite, getLatestAuditForSite, getLatestAudits, + removeAuditsForSite, } from './accessPatterns.js'; export const auditFunctions = (dynamoClient, log) => ({ @@ -48,4 +49,9 @@ export const auditFunctions = (dynamoClient, log) => ({ log, auditData, ), + removeAuditsForSite: (siteId) => removeAuditsForSite( + dynamoClient, + log, + siteId, + ), }); diff --git a/packages/spacecat-shared-data-access/src/service/sites/accessPatterns.js b/packages/spacecat-shared-data-access/src/service/sites/accessPatterns.js index 4aab3a33d..e7dde5ec6 100644 --- a/packages/spacecat-shared-data-access/src/service/sites/accessPatterns.js +++ b/packages/spacecat-shared-data-access/src/service/sites/accessPatterns.js @@ -15,7 +15,7 @@ import { isObject } from '@adobe/spacecat-shared-utils'; import { getAuditsForSite, getLatestAuditForSite, - getLatestAudits, + getLatestAudits, removeAuditsForSite, } from '../audits/accessPatterns.js'; import { createSite } from '../../models/site.js'; @@ -241,3 +241,22 @@ export const updateSite = async (dynamoClient, log, site) => { return site; }; + +/** + * Removes a site and its related audits. + * + * @param {DynamoDbClient} dynamoClient - The DynamoDB client. + * @param {Logger} log - The logger. + * @param {string} siteId - The ID of the site to remove. + * @returns {Promise} + */ +export const removeSite = async (dynamoClient, log, siteId) => { + try { + await removeAuditsForSite(dynamoClient, log, siteId); + + await dynamoClient.removeItem(TABLE_NAME_SITES, { siteId }); + } catch (error) { + log.error(`Error removing site: ${error.message}`); + throw error; + } +}; diff --git a/packages/spacecat-shared-data-access/src/service/sites/index.js b/packages/spacecat-shared-data-access/src/service/sites/index.js index 1009eccbe..f43a2cb5c 100644 --- a/packages/spacecat-shared-data-access/src/service/sites/index.js +++ b/packages/spacecat-shared-data-access/src/service/sites/index.js @@ -18,7 +18,7 @@ import { getSiteByBaseURLWithLatestAudit, getSites, getSitesToAudit, - getSitesWithLatestAudit, + getSitesWithLatestAudit, removeSite, updateSite, } from './accessPatterns.js'; @@ -61,4 +61,5 @@ export const siteFunctions = (dynamoClient, log) => ({ ), addSite: (siteData) => addSite(dynamoClient, log, siteData), updateSite: (site) => updateSite(dynamoClient, log, site), + removeSite: (siteId) => removeSite(dynamoClient, log, siteId), }); diff --git a/packages/spacecat-shared-data-access/test/index.test.js b/packages/spacecat-shared-data-access/test/index.test.js index d6c933f4a..9b9572066 100644 --- a/packages/spacecat-shared-data-access/test/index.test.js +++ b/packages/spacecat-shared-data-access/test/index.test.js @@ -22,10 +22,12 @@ describe('Data Access Object Tests', () => { 'getAuditsForSite', 'getLatestAudits', 'getLatestAuditForSite', + 'removeAuditsForSite', ]; const siteFunctions = [ 'addSite', 'updateSite', + 'removeSite', 'getSites', 'getSitesToAudit', 'getSitesWithLatestAudit', diff --git a/packages/spacecat-shared-data-access/test/service/audits/index.test.js b/packages/spacecat-shared-data-access/test/service/audits/index.test.js index 2d127b046..3dfbf707c 100644 --- a/packages/spacecat-shared-data-access/test/service/audits/index.test.js +++ b/packages/spacecat-shared-data-access/test/service/audits/index.test.js @@ -12,11 +12,16 @@ /* eslint-env mocha */ -import { expect } from 'chai'; +import chai from 'chai'; +import chaiAsPromised from 'chai-as-promised'; import sinon from 'sinon'; import { auditFunctions } from '../../../src/service/audits/index.js'; +chai.use(chaiAsPromised); + +const { expect } = chai; + describe('Audit Access Pattern Tests', () => { describe('Audit Functions Export Tests', () => { const mockDynamoClient = {}; @@ -48,8 +53,12 @@ describe('Audit Access Pattern Tests', () => { beforeEach(() => { mockDynamoClient = { query: sinon.stub().returns(Promise.resolve([])), + removeItem: sinon.stub().resolves(), + }; + mockLog = { + log: sinon.stub(), + error: sinon.stub(), }; - mockLog = { log: sinon.stub() }; exportedFunctions = auditFunctions(mockDynamoClient, mockLog); }); @@ -138,8 +147,12 @@ describe('Audit Access Pattern Tests', () => { mockDynamoClient = { query: sinon.stub().returns(Promise.resolve([])), putItem: sinon.stub().returns(Promise.resolve()), + removeItem: sinon.stub().returns(Promise.resolve()), + }; + mockLog = { + log: sinon.stub(), + error: sinon.stub(), }; - mockLog = { log: sinon.stub() }; exportedFunctions = auditFunctions(mockDynamoClient, mockLog); }); @@ -182,5 +195,47 @@ describe('Audit Access Pattern Tests', () => { await expect(exportedFunctions.addAudit(incompleteAuditData)).to.be.rejectedWith('Missing expected property'); }); + + it('should remove all audits and latest audits for a site', async () => { + const mockAuditData = [{ + siteId: 'siteId', + auditType: 'lhs', + auditedAt: new Date().toISOString(), + auditResult: { + performance: 0.9, + seo: 0.9, + accessibility: 0.9, + 'best-practices': 0.9, + }, + fullAuditRef: 'https://someurl.com', + }]; + mockDynamoClient.query.returns(Promise.resolve(mockAuditData)); + + await exportedFunctions.removeAuditsForSite('test-id'); + + expect(mockDynamoClient.query.calledTwice).to.be.true; + expect(mockDynamoClient.removeItem.calledTwice).to.be.true; + }); + + it('should log an error if the removal fails', async () => { + const mockAuditData = [{ + siteId: 'siteId', + auditType: 'lhs', + auditedAt: new Date().toISOString(), + auditResult: { + performance: 0.9, + seo: 0.9, + accessibility: 0.9, + 'best-practices': 0.9, + }, + fullAuditRef: 'https://someurl.com', + }]; + mockDynamoClient.query.returns(Promise.resolve(mockAuditData)); + + const errorMessage = 'Failed to delete item'; + mockDynamoClient.removeItem.rejects(new Error(errorMessage)); + + await expect(exportedFunctions.removeAuditsForSite('some-id')).to.be.rejectedWith(errorMessage); + }); }); }); diff --git a/packages/spacecat-shared-data-access/test/service/sites/index.test.js b/packages/spacecat-shared-data-access/test/service/sites/index.test.js index 3dcf23463..738544540 100644 --- a/packages/spacecat-shared-data-access/test/service/sites/index.test.js +++ b/packages/spacecat-shared-data-access/test/service/sites/index.test.js @@ -318,4 +318,36 @@ describe('Site Access Pattern Tests', () => { await expect(exportedFunctions.updateSite(site)).to.be.rejectedWith('Site not found'); }); }); + + describe('removeSite Tests', () => { + let mockDynamoClient; + let mockLog; + let exportedFunctions; + + beforeEach(() => { + mockDynamoClient = { + query: sinon.stub().returns(Promise.resolve([])), + removeItem: sinon.stub().returns(Promise.resolve()), + }; + mockLog = { + log: sinon.stub(), + error: sinon.stub(), + }; + exportedFunctions = siteFunctions(mockDynamoClient, mockLog); + }); + + it('removes the site and its related audits', async () => { + await exportedFunctions.removeSite('some-id'); + + expect(mockDynamoClient.removeItem.calledOnce).to.be.true; + }); + + it('logs an error and reject if the site removal fails', async () => { + const errorMessage = 'Failed to delete site'; + mockDynamoClient.removeItem.rejects(new Error(errorMessage)); + + await expect(exportedFunctions.removeSite('some-id')).to.be.rejectedWith(errorMessage); + expect(mockLog.error.calledOnce).to.be.true; + }); + }); }); From 5d0b00f62d11b4435f6354b35cd97c2ab430e6da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominique=20J=C3=A4ggi?= Date: Fri, 1 Dec 2023 12:02:58 +0100 Subject: [PATCH 24/28] fix: removeSite, add integration test --- .../src/service/audits/accessPatterns.js | 14 ++++++-------- .../src/service/sites/accessPatterns.js | 2 +- .../test-it/db.test.js | 16 ++++++++++++++++ 3 files changed, 23 insertions(+), 9 deletions(-) diff --git a/packages/spacecat-shared-data-access/src/service/audits/accessPatterns.js b/packages/spacecat-shared-data-access/src/service/audits/accessPatterns.js index e4e4439ca..a0c381c42 100644 --- a/packages/spacecat-shared-data-access/src/service/audits/accessPatterns.js +++ b/packages/spacecat-shared-data-access/src/service/audits/accessPatterns.js @@ -189,14 +189,12 @@ export const addAudit = async (dynamoClient, log, auditData) => { return audit; }; -async function removeAudits(dynamoClient, audits) { +async function removeAudits(dynamoClient, audits, latest = false) { + const tableName = latest ? TABLE_NAME_LATEST_AUDITS : TABLE_NAME_AUDITS; // TODO: use batch-remove (needs dynamo client update) - const removeAuditPromises = audits.map((audit) => dynamoClient.removeItem({ - TableName: TABLE_NAME_AUDITS, - Key: { - siteId: audit.getSiteId(), - auditedAt: audit.getAuditedAt(), - }, + const removeAuditPromises = audits.map((audit) => dynamoClient.removeItem(tableName, { + siteId: audit.getSiteId(), + SK: `${audit.getAuditType()}#${audit.getAuditedAt()}`, })); await Promise.all(removeAuditPromises); @@ -216,7 +214,7 @@ export const removeAuditsForSite = async (dynamoClient, log, siteId) => { const latestAudits = await getLatestAuditsForSite(dynamoClient, log, siteId); await removeAudits(dynamoClient, audits); - await removeAudits(dynamoClient, latestAudits); + await removeAudits(dynamoClient, latestAudits, true); } catch (error) { log.error(`Error removing audits for site ${siteId}: ${error.message}`); throw error; diff --git a/packages/spacecat-shared-data-access/src/service/sites/accessPatterns.js b/packages/spacecat-shared-data-access/src/service/sites/accessPatterns.js index e7dde5ec6..910ba0863 100644 --- a/packages/spacecat-shared-data-access/src/service/sites/accessPatterns.js +++ b/packages/spacecat-shared-data-access/src/service/sites/accessPatterns.js @@ -254,7 +254,7 @@ export const removeSite = async (dynamoClient, log, siteId) => { try { await removeAuditsForSite(dynamoClient, log, siteId); - await dynamoClient.removeItem(TABLE_NAME_SITES, { siteId }); + await dynamoClient.removeItem(TABLE_NAME_SITES, { id: siteId }); } catch (error) { log.error(`Error removing site: ${error.message}`); throw error; diff --git a/packages/spacecat-shared-data-access/test-it/db.test.js b/packages/spacecat-shared-data-access/test-it/db.test.js index 63de90031..f4d63071d 100644 --- a/packages/spacecat-shared-data-access/test-it/db.test.js +++ b/packages/spacecat-shared-data-access/test-it/db.test.js @@ -330,4 +330,20 @@ describe('DynamoDB Integration Test', async () => { // Try to add the same audit again await expect(dataAccess.addAudit(auditData)).to.be.rejectedWith('Audit already exists'); }); + + it('successfully removes a site and its related audits', async () => { + const siteToRemove = await dataAccess.getSiteByBaseURL('https://example1.com'); + const siteId = siteToRemove.getId(); + + await expect(dataAccess.removeSite(siteId)).to.eventually.be.fulfilled; + + const siteAfterRemoval = await dataAccess.getSiteByBaseURL('https://example1.com'); + expect(siteAfterRemoval).to.be.null; + + const auditsAfterRemoval = await dataAccess.getAuditsForSite(siteId); + expect(auditsAfterRemoval).to.be.an('array').that.is.empty; + + const latestAuditAfterRemoval = await dataAccess.getLatestAuditForSite(siteId, AUDIT_TYPE_LHS); + expect(latestAuditAfterRemoval).to.be.null; + }); }); From cfc2448b8a9aa835b11f17cc60a9d369840f7279 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominique=20J=C3=A4ggi?= Date: Fri, 1 Dec 2023 12:16:51 +0100 Subject: [PATCH 25/28] chore: move tests to appropriate folders --- packages/spacecat-shared-data-access/package.json | 4 ++-- .../{test-it => test/it}/auditUtils.js | 0 .../spacecat-shared-data-access/{test-it => test/it}/db.js | 0 .../{test-it => test/it}/db.test.js | 6 +++--- .../{test-it => test/it}/generateSampleData.js | 2 +- .../{test-it => test/it}/tableOperations.js | 0 .../{test-it => test/it}/util.js | 0 .../test/{ => unit}/index.test.js | 2 +- .../test/{ => unit}/models/audit.test.js | 2 +- .../test/{ => unit}/models/base.test.js | 2 +- .../test/{ => unit}/models/site.test.js | 2 +- .../test/{ => unit}/service/audits/index.test.js | 2 +- .../test/{ => unit}/service/sites/index.test.js | 4 ++-- .../spacecat-shared-data-access/test/{ => unit}/util.js | 0 14 files changed, 13 insertions(+), 13 deletions(-) rename packages/spacecat-shared-data-access/{test-it => test/it}/auditUtils.js (100%) rename packages/spacecat-shared-data-access/{test-it => test/it}/db.js (100%) rename packages/spacecat-shared-data-access/{test-it => test/it}/db.test.js (98%) rename packages/spacecat-shared-data-access/{test-it => test/it}/generateSampleData.js (98%) rename packages/spacecat-shared-data-access/{test-it => test/it}/tableOperations.js (100%) rename packages/spacecat-shared-data-access/{test-it => test/it}/util.js (100%) rename packages/spacecat-shared-data-access/test/{ => unit}/index.test.js (97%) rename packages/spacecat-shared-data-access/test/{ => unit}/models/audit.test.js (97%) rename packages/spacecat-shared-data-access/test/{ => unit}/models/base.test.js (97%) rename packages/spacecat-shared-data-access/test/{ => unit}/models/site.test.js (98%) rename packages/spacecat-shared-data-access/test/{ => unit}/service/audits/index.test.js (99%) rename packages/spacecat-shared-data-access/test/{ => unit}/service/sites/index.test.js (98%) rename packages/spacecat-shared-data-access/test/{ => unit}/util.js (100%) diff --git a/packages/spacecat-shared-data-access/package.json b/packages/spacecat-shared-data-access/package.json index 47f543f9d..15303bac3 100644 --- a/packages/spacecat-shared-data-access/package.json +++ b/packages/spacecat-shared-data-access/package.json @@ -6,8 +6,8 @@ "main": "src/index.js", "types": "src/index.d.ts", "scripts": { - "test:it": "mocha --spec \"test-it/**/*.test.js\"", - "test": "c8 mocha --spec \"test/**/*.test.js\"", + "test:it": "mocha --spec \"test/it/**/*.test.js\"", + "test": "c8 mocha --spec \"test/unit/**/*.test.js\"", "lint": "eslint .", "clean": "rm -rf package-lock.json node_modules" }, diff --git a/packages/spacecat-shared-data-access/test-it/auditUtils.js b/packages/spacecat-shared-data-access/test/it/auditUtils.js similarity index 100% rename from packages/spacecat-shared-data-access/test-it/auditUtils.js rename to packages/spacecat-shared-data-access/test/it/auditUtils.js diff --git a/packages/spacecat-shared-data-access/test-it/db.js b/packages/spacecat-shared-data-access/test/it/db.js similarity index 100% rename from packages/spacecat-shared-data-access/test-it/db.js rename to packages/spacecat-shared-data-access/test/it/db.js diff --git a/packages/spacecat-shared-data-access/test-it/db.test.js b/packages/spacecat-shared-data-access/test/it/db.test.js similarity index 98% rename from packages/spacecat-shared-data-access/test-it/db.test.js rename to packages/spacecat-shared-data-access/test/it/db.test.js index f4d63071d..504f229ba 100644 --- a/packages/spacecat-shared-data-access/test-it/db.test.js +++ b/packages/spacecat-shared-data-access/test/it/db.test.js @@ -17,9 +17,9 @@ import chaiAsPromised from 'chai-as-promised'; import dynamoDbLocal from 'dynamo-db-local'; import { isIsoDate, isValidUrl } from '@adobe/spacecat-shared-utils'; -import { sleep } from '../test/util.js'; -import { createDataAccess } from '../src/index.js'; -import { AUDIT_TYPE_LHS } from '../src/models/audit.js'; +import { sleep } from '../unit/util.js'; +import { createDataAccess } from '../../src/index.js'; +import { AUDIT_TYPE_LHS } from '../../src/models/audit.js'; import generateSampleData from './generateSampleData.js'; diff --git a/packages/spacecat-shared-data-access/test-it/generateSampleData.js b/packages/spacecat-shared-data-access/test/it/generateSampleData.js similarity index 98% rename from packages/spacecat-shared-data-access/test-it/generateSampleData.js rename to packages/spacecat-shared-data-access/test/it/generateSampleData.js index 7f74c3b4a..60719db65 100644 --- a/packages/spacecat-shared-data-access/test-it/generateSampleData.js +++ b/packages/spacecat-shared-data-access/test/it/generateSampleData.js @@ -16,7 +16,7 @@ import { dbClient, docClient as client } from './db.js'; import { generateRandomAudit } from './auditUtils.js'; import { createTable, deleteTable } from './tableOperations.js'; -import schema from '../docs/schema.json' assert { type: 'json' }; +import schema from '../../docs/schema.json' assert { type: 'json' }; /** * Creates all tables defined in a schema. diff --git a/packages/spacecat-shared-data-access/test-it/tableOperations.js b/packages/spacecat-shared-data-access/test/it/tableOperations.js similarity index 100% rename from packages/spacecat-shared-data-access/test-it/tableOperations.js rename to packages/spacecat-shared-data-access/test/it/tableOperations.js diff --git a/packages/spacecat-shared-data-access/test-it/util.js b/packages/spacecat-shared-data-access/test/it/util.js similarity index 100% rename from packages/spacecat-shared-data-access/test-it/util.js rename to packages/spacecat-shared-data-access/test/it/util.js diff --git a/packages/spacecat-shared-data-access/test/index.test.js b/packages/spacecat-shared-data-access/test/unit/index.test.js similarity index 97% rename from packages/spacecat-shared-data-access/test/index.test.js rename to packages/spacecat-shared-data-access/test/unit/index.test.js index 9b9572066..0a992aa12 100644 --- a/packages/spacecat-shared-data-access/test/index.test.js +++ b/packages/spacecat-shared-data-access/test/unit/index.test.js @@ -13,7 +13,7 @@ /* eslint-env mocha */ import { expect } from 'chai'; -import { createDataAccess } from '../src/index.js'; +import { createDataAccess } from '../../src/index.js'; describe('Data Access Object Tests', () => { const auditFunctions = [ diff --git a/packages/spacecat-shared-data-access/test/models/audit.test.js b/packages/spacecat-shared-data-access/test/unit/models/audit.test.js similarity index 97% rename from packages/spacecat-shared-data-access/test/models/audit.test.js rename to packages/spacecat-shared-data-access/test/unit/models/audit.test.js index a5fa9658d..e1a3c82d8 100644 --- a/packages/spacecat-shared-data-access/test/models/audit.test.js +++ b/packages/spacecat-shared-data-access/test/unit/models/audit.test.js @@ -13,7 +13,7 @@ /* eslint-env mocha */ import { expect } from 'chai'; -import { createAudit } from '../../src/models/audit.js'; +import { createAudit } from '../../../src/models/audit.js'; const validData = { siteId: '123', diff --git a/packages/spacecat-shared-data-access/test/models/base.test.js b/packages/spacecat-shared-data-access/test/unit/models/base.test.js similarity index 97% rename from packages/spacecat-shared-data-access/test/models/base.test.js rename to packages/spacecat-shared-data-access/test/unit/models/base.test.js index d4feb48d1..8005fff40 100644 --- a/packages/spacecat-shared-data-access/test/models/base.test.js +++ b/packages/spacecat-shared-data-access/test/unit/models/base.test.js @@ -15,7 +15,7 @@ import { isIsoDate } from '@adobe/spacecat-shared-utils'; import { expect } from 'chai'; -import { Base } from '../../src/models/base.js'; +import { Base } from '../../../src/models/base.js'; import { sleep } from '../util.js'; describe('Base Model Tests', () => { diff --git a/packages/spacecat-shared-data-access/test/models/site.test.js b/packages/spacecat-shared-data-access/test/unit/models/site.test.js similarity index 98% rename from packages/spacecat-shared-data-access/test/models/site.test.js rename to packages/spacecat-shared-data-access/test/unit/models/site.test.js index 5489f1ee6..72adcd26a 100644 --- a/packages/spacecat-shared-data-access/test/models/site.test.js +++ b/packages/spacecat-shared-data-access/test/unit/models/site.test.js @@ -13,7 +13,7 @@ /* eslint-env mocha */ import { expect } from 'chai'; -import { createSite } from '../../src/models/site.js'; +import { createSite } from '../../../src/models/site.js'; import { sleep } from '../util.js'; // Constants for testing diff --git a/packages/spacecat-shared-data-access/test/service/audits/index.test.js b/packages/spacecat-shared-data-access/test/unit/service/audits/index.test.js similarity index 99% rename from packages/spacecat-shared-data-access/test/service/audits/index.test.js rename to packages/spacecat-shared-data-access/test/unit/service/audits/index.test.js index 3dfbf707c..ec064c789 100644 --- a/packages/spacecat-shared-data-access/test/service/audits/index.test.js +++ b/packages/spacecat-shared-data-access/test/unit/service/audits/index.test.js @@ -16,7 +16,7 @@ import chai from 'chai'; import chaiAsPromised from 'chai-as-promised'; import sinon from 'sinon'; -import { auditFunctions } from '../../../src/service/audits/index.js'; +import { auditFunctions } from '../../../../src/service/audits/index.js'; chai.use(chaiAsPromised); diff --git a/packages/spacecat-shared-data-access/test/service/sites/index.test.js b/packages/spacecat-shared-data-access/test/unit/service/sites/index.test.js similarity index 98% rename from packages/spacecat-shared-data-access/test/service/sites/index.test.js rename to packages/spacecat-shared-data-access/test/unit/service/sites/index.test.js index 738544540..6082a81a5 100644 --- a/packages/spacecat-shared-data-access/test/service/sites/index.test.js +++ b/packages/spacecat-shared-data-access/test/unit/service/sites/index.test.js @@ -16,8 +16,8 @@ import chai from 'chai'; import chaiAsPromised from 'chai-as-promised'; import sinon from 'sinon'; -import { siteFunctions } from '../../../src/service/sites/index.js'; -import { createSite } from '../../../src/models/site.js'; +import { siteFunctions } from '../../../../src/service/sites/index.js'; +import { createSite } from '../../../../src/models/site.js'; chai.use(chaiAsPromised); diff --git a/packages/spacecat-shared-data-access/test/util.js b/packages/spacecat-shared-data-access/test/unit/util.js similarity index 100% rename from packages/spacecat-shared-data-access/test/util.js rename to packages/spacecat-shared-data-access/test/unit/util.js From a15ac06f9443a07d347835627fb65acaed6cd350 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominique=20J=C3=A4ggi?= Date: Fri, 1 Dec 2023 14:38:12 +0100 Subject: [PATCH 26/28] chore: types and small corrections --- .../src/dto/audit.js | 2 +- .../spacecat-shared-data-access/src/index.d.ts | 3 ++- .../src/models/audit.js | 2 +- .../src/service/audits/accessPatterns.js | 17 +++++++++++++---- .../src/service/sites/accessPatterns.js | 17 ++++++++++------- 5 files changed, 27 insertions(+), 14 deletions(-) diff --git a/packages/spacecat-shared-data-access/src/dto/audit.js b/packages/spacecat-shared-data-access/src/dto/audit.js index 156fc5645..0903e251c 100644 --- a/packages/spacecat-shared-data-access/src/dto/audit.js +++ b/packages/spacecat-shared-data-access/src/dto/audit.js @@ -51,7 +51,7 @@ export const AuditDto = { /** * Converts a DynamoDB item into an Audit object. - * @param {object } dynamoItem - DynamoDB item. + * @param {object} dynamoItem - DynamoDB item. * @returns {Readonly} Audit object. */ fromDynamoItem: (dynamoItem) => { diff --git a/packages/spacecat-shared-data-access/src/index.d.ts b/packages/spacecat-shared-data-access/src/index.d.ts index b2aef3069..09a4d509d 100644 --- a/packages/spacecat-shared-data-access/src/index.d.ts +++ b/packages/spacecat-shared-data-access/src/index.d.ts @@ -10,6 +10,8 @@ * governing permissions and limitations under the License. */ +// TODO: introduce AuditType interface or Scores interface + export interface Audit { getSiteId: () => string; getAuditedAt: () => string; @@ -27,7 +29,6 @@ export interface Site { getCreatedAt: () => string; getUpdatedAt: () => string; getAudits: () => Audit[]; - updateBaseURL: (baseURL: string) => Site; updateImsOrgId: (imsOrgId: string) => Site; setAudits: (audits: Audit[]) => Site; } diff --git a/packages/spacecat-shared-data-access/src/models/audit.js b/packages/spacecat-shared-data-access/src/models/audit.js index 1f4b9ef81..6b9d77f34 100644 --- a/packages/spacecat-shared-data-access/src/models/audit.js +++ b/packages/spacecat-shared-data-access/src/models/audit.js @@ -66,7 +66,7 @@ const Audit = (data = {}) => { /** * Creates a new Audit. * - * @param {object } data - audit data + * @param {object} data - audit data * @returns {Readonly} audit - new audit */ export const createAudit = (data) => { diff --git a/packages/spacecat-shared-data-access/src/service/audits/accessPatterns.js b/packages/spacecat-shared-data-access/src/service/audits/accessPatterns.js index a0c381c42..df611922d 100644 --- a/packages/spacecat-shared-data-access/src/service/audits/accessPatterns.js +++ b/packages/spacecat-shared-data-access/src/service/audits/accessPatterns.js @@ -28,7 +28,8 @@ const PK_ALL_LATEST_AUDITS = 'ALL_LATEST_AUDITS'; * @param {Logger} log - The logger. * @param {string} siteId - The ID of the site for which audits are being retrieved. * @param {string} [auditType] - Optional. The type of audits to retrieve. - * @returns {Promise} A promise that resolves to an array of audits for the specified site. + * @returns {Promise[]>} A promise that resolves to an array of audits + * for the specified site. */ export const getAuditsForSite = async (dynamoClient, log, siteId, auditType) => { const queryParams = { @@ -87,7 +88,7 @@ export const getAuditForSite = async ( * @param {string} auditType - The type of audits to retrieve. * @param {boolean} ascending - Determines if the audits should be sorted ascending * or descending by scores. - * @returns {Promise} A promise that resolves to an array of the latest + * @returns {Promise[]>} A promise that resolves to an array of the latest * audits of the specified type. */ export const getLatestAudits = async ( @@ -116,7 +117,7 @@ export const getLatestAudits = async ( * @param {DynamoDbClient} dynamoClient - The DynamoDB client. * @param {Logger} log - The logger. * @param {string} siteId - The ID of the site for which audits are being retrieved. - * @returns {Promise} A promise that resolves to an array of latest audits + * @returns {Promise[]>} A promise that resolves to an array of latest audits * for the specified site. */ export const getLatestAuditsForSite = async (dynamoClient, log, siteId) => { @@ -166,7 +167,7 @@ export const getLatestAuditForSite = async ( * @param {DynamoDbClient} dynamoClient - The DynamoDB client. * @param {Logger} log - The logger. * @param {object} auditData - The audit data. - * @returns {Promise>} + * @returns {Promise>} */ export const addAudit = async (dynamoClient, log, auditData) => { const audit = createAudit(auditData); @@ -189,6 +190,14 @@ export const addAudit = async (dynamoClient, log, auditData) => { return audit; }; +/** + * Removes audits from the database. + * @param {DynamoDbClient} dynamoClient - The DynamoDB client. + * @param {Logger} log - The logger. + * @param audits + * @param latest + * @returns {Promise} + */ async function removeAudits(dynamoClient, audits, latest = false) { const tableName = latest ? TABLE_NAME_LATEST_AUDITS : TABLE_NAME_AUDITS; // TODO: use batch-remove (needs dynamo client update) diff --git a/packages/spacecat-shared-data-access/src/service/sites/accessPatterns.js b/packages/spacecat-shared-data-access/src/service/sites/accessPatterns.js index 910ba0863..f5bcf99c6 100644 --- a/packages/spacecat-shared-data-access/src/service/sites/accessPatterns.js +++ b/packages/spacecat-shared-data-access/src/service/sites/accessPatterns.js @@ -29,7 +29,7 @@ const TABLE_NAME_SITES = 'sites'; * Retrieves all sites. * * @param {DynamoDbClient} dynamoClient - The DynamoDB client. - * @returns {Promise>} A promise that resolves to an array of all sites. + * @returns {Promise[]>} A promise that resolves to an array of all sites. */ export const getSites = async (dynamoClient) => { const dynamoItems = await dynamoClient.query({ @@ -64,7 +64,7 @@ export const getSitesToAudit = async (dynamoClient) => { * @param {string} auditType - The type of the latest audits to retrieve for each site. * @param {boolean} [sortAuditsAscending] - Optional. Determines if the audits * should be sorted ascending or descending by scores. - * @returns {Promise} A promise that resolves to an array of site objects, + * @returns {Promise[]>} A promise that resolves to an array of site objects, * each with its latest audit of the specified type. */ export const getSitesWithLatestAudit = async ( @@ -96,7 +96,7 @@ export const getSitesWithLatestAudit = async ( * @param {DynamoDbClient} dynamoClient - The DynamoDB client. * @param {Logger} log - The logger. * @param {string} baseURL - The base URL of the site to retrieve. - * @returns {Promise} A promise that resolves to the site object if found, + * @returns {Promise|null>} A promise that resolves to the site object if found, * otherwise null. */ export const getSiteByBaseURL = async ( @@ -130,7 +130,7 @@ export const getSiteByBaseURL = async ( * @param {string} baseUrl - The base URL of the site to retrieve. * @param {string} auditType - The type of audits to retrieve for the site. * @param {boolean} [latestOnly=false] - Determines if only the latest audit should be retrieved. - * @returns {Promise} A promise that resolves to the site object with audit + * @returns {Promise|null>} A promise that resolves to the site object with audit * data if found, otherwise null. */ export const getSiteByBaseURLWithAuditInfo = async ( @@ -172,7 +172,8 @@ export const getSiteByBaseURLWithAuditInfo = async ( * @param {Logger} log - The logger. * @param {string} baseUrl - The base URL of the site to retrieve. * @param {string} auditType - The type of audits to retrieve for the site. - * @returns {Promise} A promise that resolves to the site object with all its audits. + * @returns {Promise|null>} A promise that resolves to the site object + * with all its audits. */ export const getSiteByBaseURLWithAudits = async ( dynamoClient, @@ -188,7 +189,8 @@ export const getSiteByBaseURLWithAudits = async ( * @param {Logger} log - The logger. * @param {string} baseUrl - The base URL of the site to retrieve. * @param {string} auditType - The type of the latest audit to retrieve for the site. - * @returns {Promise} A promise that resolves to the site object with its latest audit. + * @returns {Promise|null>} A promise that resolves to the site object + * with its latest audit. */ export const getSiteByBaseURLWithLatestAudit = async ( dynamoClient, @@ -228,7 +230,7 @@ export const addSite = async (dynamoClient, log, siteData) => { * @param {DynamoDbClient} dynamoClient - The DynamoDB client. * @param {Logger} log - The logger. * @param {Site} site - The site. - * @returns {Promise} - The updated site. + * @returns {Promise>} - The updated site. */ export const updateSite = async (dynamoClient, log, site) => { const existingSite = await getSiteByBaseURL(dynamoClient, log, site.getBaseURL()); @@ -252,6 +254,7 @@ export const updateSite = async (dynamoClient, log, site) => { */ export const removeSite = async (dynamoClient, log, siteId) => { try { + // TODO: Add transaction support await removeAuditsForSite(dynamoClient, log, siteId); await dynamoClient.removeItem(TABLE_NAME_SITES, { id: siteId }); From e33e1a5a27986ec7130586d2d24f0e4c2cefbd3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominique=20J=C3=A4ggi?= Date: Fri, 1 Dec 2023 15:17:44 +0100 Subject: [PATCH 27/28] feat: add wrapper, update doc --- .../spacecat-shared-data-access/README.md | 72 +++++++++++++++++++ .../spacecat-shared-data-access/package.json | 2 +- .../spacecat-shared-data-access/src/index.js | 27 +++---- .../src/service/index.js | 33 +++++++++ .../test/it/db.test.js | 2 +- .../test/unit/index.test.js | 67 +++++++---------- .../test/unit/service/index.test.js | 64 +++++++++++++++++ 7 files changed, 207 insertions(+), 60 deletions(-) create mode 100644 packages/spacecat-shared-data-access/src/service/index.js create mode 100644 packages/spacecat-shared-data-access/test/unit/service/index.test.js diff --git a/packages/spacecat-shared-data-access/README.md b/packages/spacecat-shared-data-access/README.md index 588bf8b79..3fb6fe1fe 100644 --- a/packages/spacecat-shared-data-access/README.md +++ b/packages/spacecat-shared-data-access/README.md @@ -71,6 +71,78 @@ The module provides two main DAOs: - `getLatestAuditForSite` - `addAudit` + +## Integrating Data Access in AWS Lambda Functions + +Our `spacecat-shared-data-access` module includes a wrapper that can be easily integrated into AWS Lambda functions using `@adobe/helix-shared-wrap`. This integration allows your Lambda functions to access and manipulate data seamlessly. + +### Steps for Integration + +1. **Import the Data Access Wrapper** + + Along with other wrappers and utilities, import the `dataAccessWrapper`. + + ```javascript + import dataAccessWrapper from '@adobe/spacecat-shared-data-access/wrapper'; + ``` + +2. **Modify Your Lambda Wrapper Script** + + Include `dataAccessWrapper` in the chain of wrappers when defining your Lambda handler. + + ```javascript + export const main = wrap(run) + .with(sqsEventAdapter) + .with(dataAccessWrapper) // Add this line + .with(sqs) + .with(secrets) + .with(helixStatus); + ``` + +3. **Access Data in Your Lambda Function** + + Use the `dataAccess` object from the context to interact with your data layer. + + ```javascript + async function run(message, context) { + const { dataAccess } = context; + + // Example: Retrieve all sites + const sites = await dataAccess.getSites(); + // ... more logic ... + } + ``` + +### Example + +Here's a complete example of a Lambda function utilizing the data access wrapper: + +```javascript +import wrap from '@adobe/helix-shared-wrap'; +import dataAccessWrapper from '@adobe/spacecat-shared-data-access/wrapper'; +import sqsEventAdapter from './sqsEventAdapter'; +import sqs from './sqs'; +import secrets from '@adobe/helix-shared-secrets'; +import helixStatus from '@adobe/helix-status'; + +async function run(message, context) { + const { dataAccess } = context; + try { + const sites = await dataAccess.getSites(); + // Function logic here + } catch (error) { + // Error handling + } +} + +export const main = wrap(run) + .with(sqsEventAdapter) + .with(dataAccessWrapper) + .with(sqs) + .with(secrets) + .with(helixStatus); +``` + ## Contributing Contributions to `spacecat-shared-data-access` are welcome. Please adhere to the standard Git workflow and submit pull requests for proposed changes. diff --git a/packages/spacecat-shared-data-access/package.json b/packages/spacecat-shared-data-access/package.json index 15303bac3..327071b69 100644 --- a/packages/spacecat-shared-data-access/package.json +++ b/packages/spacecat-shared-data-access/package.json @@ -3,7 +3,7 @@ "version": "1.1.0", "description": "Shared modules of the Spacecat Services - Data Access", "type": "module", - "main": "src/index.js", + "main": "src/service/index.js", "types": "src/index.d.ts", "scripts": { "test:it": "mocha --spec \"test/it/**/*.test.js\"", diff --git a/packages/spacecat-shared-data-access/src/index.js b/packages/spacecat-shared-data-access/src/index.js index 2b6006e82..3fab17cb4 100644 --- a/packages/spacecat-shared-data-access/src/index.js +++ b/packages/spacecat-shared-data-access/src/index.js @@ -10,24 +10,15 @@ * governing permissions and limitations under the License. */ -import { createClient } from '@adobe/spacecat-shared-dynamo'; -import { auditFunctions } from './service/audits/index.js'; -import { siteFunctions } from './service/sites/index.js'; +import { createDataAccess } from './service/index.js'; -/** - * Creates a data access object. - * - * @param {Logger} log logger - * @returns {object} data access object - */ -export const createDataAccess = (log = console) => { - const dynamoClient = createClient(log); - - const auditFuncs = auditFunctions(dynamoClient, log); - const siteFuncs = siteFunctions(dynamoClient, log); +export default function dataAccessWrapper(fn) { + return async (request, context) => { + if (!context.dataAccess) { + const { log } = context; + context.dataAccess = createDataAccess(log); + } - return { - ...auditFuncs, - ...siteFuncs, + return fn(request, context); }; -}; +} diff --git a/packages/spacecat-shared-data-access/src/service/index.js b/packages/spacecat-shared-data-access/src/service/index.js new file mode 100644 index 000000000..6f0b9bddf --- /dev/null +++ b/packages/spacecat-shared-data-access/src/service/index.js @@ -0,0 +1,33 @@ +/* + * Copyright 2023 Adobe. All rights reserved. + * This file is licensed to you under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS + * OF ANY KIND, either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ + +import { createClient } from '@adobe/spacecat-shared-dynamo'; +import { auditFunctions } from './audits/index.js'; +import { siteFunctions } from './sites/index.js'; + +/** + * Creates a data access object. + * + * @param {Logger} log logger + * @returns {object} data access object + */ +export const createDataAccess = (log = console) => { + const dynamoClient = createClient(log); + + const auditFuncs = auditFunctions(dynamoClient, log); + const siteFuncs = siteFunctions(dynamoClient, log); + + return { + ...auditFuncs, + ...siteFuncs, + }; +}; diff --git a/packages/spacecat-shared-data-access/test/it/db.test.js b/packages/spacecat-shared-data-access/test/it/db.test.js index 504f229ba..102842a18 100644 --- a/packages/spacecat-shared-data-access/test/it/db.test.js +++ b/packages/spacecat-shared-data-access/test/it/db.test.js @@ -18,7 +18,7 @@ import dynamoDbLocal from 'dynamo-db-local'; import { isIsoDate, isValidUrl } from '@adobe/spacecat-shared-utils'; import { sleep } from '../unit/util.js'; -import { createDataAccess } from '../../src/index.js'; +import { createDataAccess } from '../../src/service/index.js'; import { AUDIT_TYPE_LHS } from '../../src/models/audit.js'; import generateSampleData from './generateSampleData.js'; diff --git a/packages/spacecat-shared-data-access/test/unit/index.test.js b/packages/spacecat-shared-data-access/test/unit/index.test.js index 0a992aa12..893ca5042 100644 --- a/packages/spacecat-shared-data-access/test/unit/index.test.js +++ b/packages/spacecat-shared-data-access/test/unit/index.test.js @@ -13,52 +13,39 @@ /* eslint-env mocha */ import { expect } from 'chai'; -import { createDataAccess } from '../../src/index.js'; +import sinon from 'sinon'; +import dataAccessWrapper from '../../src/index.js'; + +describe('Data Access Wrapper Tests', () => { + let mockFn; + let mockContext; + let mockRequest; + + beforeEach(() => { + mockFn = sinon.stub().resolves('function response'); + mockContext = { log: sinon.spy() }; + mockRequest = {}; + }); -describe('Data Access Object Tests', () => { - const auditFunctions = [ - 'addAudit', - 'getAuditForSite', - 'getAuditsForSite', - 'getLatestAudits', - 'getLatestAuditForSite', - 'removeAuditsForSite', - ]; - const siteFunctions = [ - 'addSite', - 'updateSite', - 'removeSite', - 'getSites', - 'getSitesToAudit', - 'getSitesWithLatestAudit', - 'getSiteByBaseURL', - 'getSiteByBaseURLWithAuditInfo', - 'getSiteByBaseURLWithAudits', - 'getSiteByBaseURLWithLatestAudit', - ]; + afterEach(() => { + sinon.restore(); + }); - let dao; + it('adds dataAccess to context and calls the wrapped function', async () => { + const wrappedFn = dataAccessWrapper(mockFn); - before(() => { - dao = createDataAccess(); - }); + const response = await wrappedFn(mockRequest, mockContext); - it('contains all known audit functions', () => { - auditFunctions.forEach((funcName) => { - expect(dao).to.have.property(funcName); - }); + expect(mockFn.calledOnceWithExactly(mockRequest, mockContext)).to.be.true; + expect(response).to.equal('function response'); }); - it('contains all known site functions', () => { - siteFunctions.forEach((funcName) => { - expect(dao).to.have.property(funcName); - }); - }); + it('does not recreate dataAccess if already present in context', async () => { + mockContext.dataAccess = { existingDataAccess: true }; + const wrappedFn = dataAccessWrapper(mockFn); + + await wrappedFn(mockRequest, mockContext); - it('does not contain any unexpected functions', () => { - const expectedFunctions = new Set([...auditFunctions, ...siteFunctions]); - Object.keys(dao).forEach((funcName) => { - expect(expectedFunctions).to.include(funcName); - }); + expect(mockContext.dataAccess).to.deep.equal({ existingDataAccess: true }); }); }); diff --git a/packages/spacecat-shared-data-access/test/unit/service/index.test.js b/packages/spacecat-shared-data-access/test/unit/service/index.test.js new file mode 100644 index 000000000..2c2f537ed --- /dev/null +++ b/packages/spacecat-shared-data-access/test/unit/service/index.test.js @@ -0,0 +1,64 @@ +/* + * Copyright 2023 Adobe. All rights reserved. + * This file is licensed to you under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS + * OF ANY KIND, either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ + +/* eslint-env mocha */ + +import { expect } from 'chai'; +import { createDataAccess } from '../../../src/service/index.js'; + +describe('Data Access Object Tests', () => { + const auditFunctions = [ + 'addAudit', + 'getAuditForSite', + 'getAuditsForSite', + 'getLatestAudits', + 'getLatestAuditForSite', + 'removeAuditsForSite', + ]; + const siteFunctions = [ + 'addSite', + 'updateSite', + 'removeSite', + 'getSites', + 'getSitesToAudit', + 'getSitesWithLatestAudit', + 'getSiteByBaseURL', + 'getSiteByBaseURLWithAuditInfo', + 'getSiteByBaseURLWithAudits', + 'getSiteByBaseURLWithLatestAudit', + ]; + + let dao; + + before(() => { + dao = createDataAccess(); + }); + + it('contains all known audit functions', () => { + auditFunctions.forEach((funcName) => { + expect(dao).to.have.property(funcName); + }); + }); + + it('contains all known site functions', () => { + siteFunctions.forEach((funcName) => { + expect(dao).to.have.property(funcName); + }); + }); + + it('does not contain any unexpected functions', () => { + const expectedFunctions = new Set([...auditFunctions, ...siteFunctions]); + Object.keys(dao).forEach((funcName) => { + expect(expectedFunctions).to.include(funcName); + }); + }); +}); From 5d50b16d4342e5f277e7e324e53e3d63689ce95b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominique=20J=C3=A4ggi?= Date: Fri, 1 Dec 2023 15:23:19 +0100 Subject: [PATCH 28/28] chore: doc --- packages/spacecat-shared-data-access/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/spacecat-shared-data-access/README.md b/packages/spacecat-shared-data-access/README.md index 3fb6fe1fe..65e91a262 100644 --- a/packages/spacecat-shared-data-access/README.md +++ b/packages/spacecat-shared-data-access/README.md @@ -1,6 +1,6 @@ # SpaceCat Shared Data Access -This Node.js module, `spacecat-shared-data-access`, is a comprehensive data access layer for managing sites and their audits, leveraging Amazon DynamoDB. It's tailored for the `StarCatalogue` model, ensuring efficient querying and robust data manipulation. +This Node.js module, `spacecat-shared-data-access`, is a data access layer for managing sites and their audits, leveraging Amazon DynamoDB. ## Installation @@ -63,6 +63,7 @@ The module provides two main DAOs: - `getSiteByBaseURLWithLatestAudit` - `addSite` - `updateSite` +- `removeSite` ### Audit Functions - `getAuditsForSite`