diff --git a/packages/foundation/core/package.json b/packages/foundation/core/package.json index 8ff7a95a..20b5aee3 100644 --- a/packages/foundation/core/package.json +++ b/packages/foundation/core/package.json @@ -30,6 +30,8 @@ }, "dependencies": { "@objectql/plugin-formula": "workspace:*", + "@objectql/plugin-optimizations": "workspace:*", + "@objectql/plugin-query": "workspace:*", "@objectql/plugin-validator": "workspace:*", "@objectql/types": "workspace:*", "@objectstack/core": "^2.0.6", diff --git a/packages/foundation/core/src/index.ts b/packages/foundation/core/src/index.ts index 173f25ac..1ac63507 100644 --- a/packages/foundation/core/src/index.ts +++ b/packages/foundation/core/src/index.ts @@ -6,7 +6,7 @@ * LICENSE file in the root directory of this source tree. */ -// Re-export types from @objectstack packages for API compatibility +// ── Re-export upstream canonical engine ── export type { ObjectKernel } from '@objectstack/runtime'; export type { ObjectStackProtocolImplementation } from '@objectstack/objectql'; @@ -21,6 +21,15 @@ export { } from '@objectstack/objectql'; export type { ObjectContributor } from '@objectstack/objectql'; +// Re-export upstream types exported by @objectstack/objectql for plugin authors +export type { + ObjectQLHostContext, + HookHandler as UpstreamHookHandler, + HookEntry, + OperationContext, + EngineMiddleware, +} from '@objectstack/objectql'; + // Export ObjectStack spec types for driver development import { Data, Automation } from '@objectstack/spec'; import { z } from 'zod'; @@ -33,21 +42,74 @@ export type StateMachineConfig = z.infer; export type ObjectOwnership = z.infer; export type ObjectExtension = z.infer; +// ── Convenience factory ── +export { createObjectQLKernel, type ObjectQLKernelOptions } from './kernel-factory'; + +// ── Gateway (kept in core — upstream server handles API layer) ── export * from './gateway'; -// Export our enhanced runtime components (actual implementations) +// ── Core runtime components (backward compatibility) ── export * from './repository'; export * from './app'; export * from './plugin'; -// Export query-specific modules (ObjectQL core competency) -export * from './query'; - -// Export utilities +// ── Utilities ── export * from './util'; -// Export kernel optimizations -export * from './optimizations'; - -// Export AI runtime +// ── AI runtime (kept in core — separate AI project) ── export * from './ai'; + +// ── Re-export from @objectql/plugin-query (backward compatibility) ── +// Import from '@objectql/plugin-query' directly for new code. + +/** @deprecated Import from '@objectql/plugin-query' instead */ +export { QueryService } from '@objectql/plugin-query'; +/** @deprecated Import from '@objectql/plugin-query' instead */ +export { QueryBuilder } from '@objectql/plugin-query'; +/** @deprecated Import from '@objectql/plugin-query' instead */ +export { QueryAnalyzer } from '@objectql/plugin-query'; +/** @deprecated Import from '@objectql/plugin-query' instead */ +export { FilterTranslator } from '@objectql/plugin-query'; +/** @deprecated Import from '@objectql/plugin-query' instead */ +export { QueryPlugin } from '@objectql/plugin-query'; + +export type { + QueryOptions, + QueryResult, + QueryProfile, +} from '@objectql/plugin-query'; +export type { + QueryPlan, + ProfileResult, + QueryStats, +} from '@objectql/plugin-query'; + +// ── Re-export from @objectql/plugin-optimizations (backward compatibility) ── +// Import from '@objectql/plugin-optimizations' directly for new code. + +/** @deprecated Import from '@objectql/plugin-optimizations' instead */ +export { OptimizedMetadataRegistry } from '@objectql/plugin-optimizations'; +/** @deprecated Import from '@objectql/plugin-optimizations' instead */ +export { QueryCompiler } from '@objectql/plugin-optimizations'; +/** @deprecated Import from '@objectql/plugin-optimizations' instead */ +export { CompiledHookManager } from '@objectql/plugin-optimizations'; +/** @deprecated Import from '@objectql/plugin-optimizations' instead */ +export { GlobalConnectionPool } from '@objectql/plugin-optimizations'; +/** @deprecated Import from '@objectql/plugin-optimizations' instead */ +export { OptimizedValidationEngine } from '@objectql/plugin-optimizations'; +/** @deprecated Import from '@objectql/plugin-optimizations' instead */ +export { LazyMetadataLoader } from '@objectql/plugin-optimizations'; +/** @deprecated Import from '@objectql/plugin-optimizations' instead */ +export { DependencyGraph } from '@objectql/plugin-optimizations'; +/** @deprecated Import from '@objectql/plugin-optimizations' instead */ +export { SQLQueryOptimizer } from '@objectql/plugin-optimizations'; +/** @deprecated Import from '@objectql/plugin-optimizations' instead */ +export { OptimizationsPlugin } from '@objectql/plugin-optimizations'; + +export type { CompiledQuery } from '@objectql/plugin-optimizations'; +export type { Hook } from '@objectql/plugin-optimizations'; +export type { Connection, PoolLimits } from '@objectql/plugin-optimizations'; +export type { ValidatorFunction, ValidationSchema } from '@objectql/plugin-optimizations'; +export type { ObjectMetadata, MetadataLoader } from '@objectql/plugin-optimizations'; +export type { DependencyEdge, DependencyType } from '@objectql/plugin-optimizations'; +export type { IndexMetadata, SchemaWithIndexes, OptimizableQueryAST } from '@objectql/plugin-optimizations'; diff --git a/packages/foundation/core/src/kernel-factory.ts b/packages/foundation/core/src/kernel-factory.ts new file mode 100644 index 00000000..b66b18d0 --- /dev/null +++ b/packages/foundation/core/src/kernel-factory.ts @@ -0,0 +1,47 @@ +/** + * ObjectQL Kernel Factory + * Copyright (c) 2026-present ObjectStack Inc. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +import { ObjectKernel } from '@objectstack/runtime'; +import { ObjectQLPlugin as UpstreamObjectQLPlugin } from '@objectstack/objectql'; +import type { Plugin } from '@objectstack/core'; + +/** + * Options for creating an ObjectQL Kernel + */ +export interface ObjectQLKernelOptions { + /** + * Additional plugins to register with the kernel + */ + plugins?: Plugin[]; +} + +/** + * Convenience factory for creating an ObjectQL-ready kernel. + * + * Creates an ObjectStackKernel pre-configured with the upstream ObjectQLPlugin + * (data engine, schema registry, protocol implementation) plus any additional + * plugins provided. + * + * @example + * ```typescript + * import { createObjectQLKernel } from '@objectql/core'; + * import { QueryPlugin } from '@objectql/plugin-query'; + * import { OptimizationsPlugin } from '@objectql/plugin-optimizations'; + * + * const kernel = createObjectQLKernel({ + * plugins: [new QueryPlugin(), new OptimizationsPlugin()], + * }); + * await kernel.start(); + * ``` + */ +export function createObjectQLKernel(options: ObjectQLKernelOptions = {}): ObjectKernel { + return new (ObjectKernel as any)([ + new UpstreamObjectQLPlugin(), + ...(options.plugins || []), + ]); +} diff --git a/packages/foundation/core/src/plugin.ts b/packages/foundation/core/src/plugin.ts index d9ae0879..02bdaba8 100644 --- a/packages/foundation/core/src/plugin.ts +++ b/packages/foundation/core/src/plugin.ts @@ -11,30 +11,12 @@ import { ConsoleLogger, ObjectQLError } from '@objectql/types'; import type { Logger } from '@objectql/types'; import { ValidatorPlugin, ValidatorPluginConfig } from '@objectql/plugin-validator'; import { FormulaPlugin, FormulaPluginConfig } from '@objectql/plugin-formula'; -import { QueryService } from './query/query-service'; -import { QueryAnalyzer } from './query/query-analyzer'; +import { QueryPlugin } from '@objectql/plugin-query'; import { ObjectStackProtocolImplementation } from './protocol'; import type { Driver } from '@objectql/types'; import { createDefaultAiRegistry } from './ai'; import { SchemaRegistry } from '@objectstack/objectql'; -/** - * Extended kernel with ObjectQL services - */ -interface ExtendedKernel { - metadata?: any; - actions?: any; - hooks?: any; - getAllDrivers?: () => Driver[]; - create?: (objectName: string, data: any) => Promise; - update?: (objectName: string, id: string, data: any) => Promise; - delete?: (objectName: string, id: string) => Promise; - find?: (objectName: string, query: any) => Promise; - get?: (objectName: string, id: string) => Promise; - queryService?: QueryService; - queryAnalyzer?: QueryAnalyzer; -} - /** * Configuration for the ObjectQL Plugin */ @@ -91,12 +73,15 @@ export interface ObjectQLPluginConfig { /** * ObjectQL Plugin * - * Implements the RuntimePlugin interface to provide ObjectQL's enhanced features - * (Repository, Validator, Formula, AI) on top of the microkernel. + * Thin orchestrator that composes ObjectQL's extension plugins + * (QueryPlugin, ValidatorPlugin, FormulaPlugin, AI) on top of the microkernel. + * + * Delegates query execution to @objectql/plugin-query and provides + * repository-pattern CRUD bridging to the kernel. */ export class ObjectQLPlugin implements RuntimePlugin { name = '@objectql/core'; - version = '4.0.2'; + version = '4.2.0'; private logger: Logger; constructor(private config: ObjectQLPluginConfig = {}, _ql?: any) { @@ -119,7 +104,7 @@ export class ObjectQLPlugin implements RuntimePlugin { async install(ctx: RuntimeContext): Promise { this.logger.info('Installing plugin...'); - const kernel = ctx.engine as ExtendedKernel; + const kernel = ctx.engine as any; // Get datasources - either from config or from kernel drivers let datasources = this.config.datasources; @@ -141,21 +126,11 @@ export class ObjectQLPlugin implements RuntimePlugin { } } - // Register QueryService and QueryAnalyzer if enabled + // Delegate query service registration to QueryPlugin if (this.config.enableQueryService !== false && datasources) { - const queryService = new QueryService( - datasources, - kernel.metadata - ); - kernel.queryService = queryService; - - const queryAnalyzer = new QueryAnalyzer( - queryService, - kernel.metadata - ); - kernel.queryAnalyzer = queryAnalyzer; - - this.logger.info('QueryService and QueryAnalyzer registered'); + const queryPlugin = new QueryPlugin({ datasources }); + await queryPlugin.install(ctx); + this.logger.info('QueryPlugin installed (QueryService + QueryAnalyzer)'); } // Register components based on configuration @@ -280,8 +255,6 @@ export class ObjectQLPlugin implements RuntimePlugin { kernel.count = async (objectName: string, filters?: any): Promise => { // Use QueryService if available if ((kernel as any).queryService) { - // QueryService.count expects a UnifiedQuery filter or just filter object? - // Looking at QueryService.count signature: count(objectName: string, where?: Filter, options?: QueryOptions) const result = await (kernel as any).queryService.count(objectName, filters); return result.value; } diff --git a/packages/foundation/core/test/plugin-integration.test.ts b/packages/foundation/core/test/plugin-integration.test.ts index 0f0bc043..f26cf87e 100644 --- a/packages/foundation/core/test/plugin-integration.test.ts +++ b/packages/foundation/core/test/plugin-integration.test.ts @@ -46,7 +46,7 @@ describe('ObjectQLPlugin Integration', () => { it('should have correct name and version', () => { plugin = new ObjectQLPlugin(); expect(plugin.name).toBe('@objectql/core'); - expect(plugin.version).toBe('4.0.2'); + expect(plugin.version).toBe('4.2.0'); }); }); diff --git a/packages/foundation/core/tsconfig.json b/packages/foundation/core/tsconfig.json index 90460abb..fea745e5 100644 --- a/packages/foundation/core/tsconfig.json +++ b/packages/foundation/core/tsconfig.json @@ -12,6 +12,8 @@ "references": [ { "path": "../types" }, { "path": "../plugin-validator" }, - { "path": "../plugin-formula" } + { "path": "../plugin-formula" }, + { "path": "../plugin-query" }, + { "path": "../plugin-optimizations" } ] } diff --git a/packages/foundation/plugin-optimizations/package.json b/packages/foundation/plugin-optimizations/package.json new file mode 100644 index 00000000..46a02a3a --- /dev/null +++ b/packages/foundation/plugin-optimizations/package.json @@ -0,0 +1,27 @@ +{ + "name": "@objectql/plugin-optimizations", + "version": "4.2.0", + "description": "Performance optimization plugins for ObjectQL - connection pooling, LRU cache, compiled hooks", + "main": "dist/index.js", + "types": "dist/index.d.ts", + "sideEffects": false, + "exports": { + ".": { + "types": "./dist/index.d.ts", + "default": "./dist/index.js" + } + }, + "files": ["dist"], + "scripts": { + "build": "tsc", + "test": "vitest run" + }, + "dependencies": { + "@objectql/types": "workspace:*", + "@objectstack/core": "^2.0.6", + "@objectstack/spec": "^2.0.6" + }, + "devDependencies": { + "typescript": "^5.3.0" + } +} diff --git a/packages/foundation/plugin-optimizations/src/CompiledHookManager.ts b/packages/foundation/plugin-optimizations/src/CompiledHookManager.ts new file mode 100644 index 00000000..68605dea --- /dev/null +++ b/packages/foundation/plugin-optimizations/src/CompiledHookManager.ts @@ -0,0 +1,193 @@ +/** + * ObjectQL + * Copyright (c) 2026-present ObjectStack Inc. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +import { Logger, ConsoleLogger } from '@objectql/types'; + +/** + * Hook definition + */ +export interface Hook { + pattern: string; + handler: (context: any) => Promise | void; + packageName?: string; + priority?: number; +} + +/** + * Compiled Hook Manager + * + * Improvement: Pre-compiles hook pipelines by event pattern at registration time. + * No runtime pattern matching required. + * + * Expected: 5x faster hook execution, parallel async support + */ +export class CompiledHookManager { + // Direct event -> hooks mapping (no pattern matching at runtime) + private pipelines = new Map(); + + // Keep track of all registered hooks for management + private allHooks = new Map(); + + // Structured logger + private logger: Logger = new ConsoleLogger({ name: '@objectql/hook-manager', level: 'warn' }); + + /** + * Expand a pattern like "before*" to all matching events + */ + private expandPattern(pattern: string): string[] { + // Common event patterns + const eventTypes = [ + 'beforeCreate', 'afterCreate', + 'beforeUpdate', 'afterUpdate', + 'beforeDelete', 'afterDelete', + 'beforeFind', 'afterFind', + 'beforeCount', 'afterCount' + ]; + + // Handle wildcards + if (pattern === '*') { + return eventTypes; + } + + if (pattern.includes('*')) { + // Use global replace to handle all occurrences of * + const regex = new RegExp('^' + pattern.replace(/\*/g, '.*') + '$'); + return eventTypes.filter(event => regex.test(event)); + } + + // Exact match + return [pattern]; + } + + /** + * Register a hook - pre-groups by event pattern + */ + registerHook(event: string, objectName: string, handler: any, packageName?: string): void { + const hook: Hook = { + pattern: `${event}:${objectName}`, + handler, + packageName, + priority: 0 + }; + + // Store in all hooks registry + const hookId = `${event}:${objectName}:${Date.now()}`; + this.allHooks.set(hookId, hook); + + // Expand event patterns + const events = this.expandPattern(event); + + // Handle wildcard object names + if (objectName === '*') { + for (const concreteEvent of events) { + // Register for all potential object names + // Since we don't know all object names upfront, we keep a special '*' pipeline + const wildcardKey = `${concreteEvent}:*`; + if (!this.pipelines.has(wildcardKey)) { + this.pipelines.set(wildcardKey, []); + } + this.pipelines.get(wildcardKey)!.push(hook); + } + } else { + // Pre-group hooks by concrete event names (only for non-wildcard objects) + for (const concreteEvent of events) { + const key = `${concreteEvent}:${objectName}`; + if (!this.pipelines.has(key)) { + this.pipelines.set(key, []); + } + this.pipelines.get(key)!.push(hook); + } + } + } + + /** + * Run hooks for an event - direct lookup, no pattern matching + */ + async runHooks(event: string, objectName: string, context: any): Promise { + const key = `${event}:${objectName}`; + const wildcardKey = `${event}:*`; + + // Collect all applicable hooks + const hooks: Hook[] = []; + + // Add object-specific hooks + const objectHooks = this.pipelines.get(key); + if (objectHooks) { + hooks.push(...objectHooks); + } + + // Add wildcard hooks + const wildcardHooks = this.pipelines.get(wildcardKey); + if (wildcardHooks) { + hooks.push(...wildcardHooks); + } + + if (hooks.length === 0) { + return; + } + + // Sort by priority (higher priority first) + hooks.sort((a, b) => (b.priority || 0) - (a.priority || 0)); + + // Execute hooks in parallel for better performance + // Note: If order matters, change to sequential execution + await Promise.all(hooks.map(hook => { + try { + return Promise.resolve(hook.handler(context)); + } catch (error) { + this.logger.error(`Hook execution failed for ${event}:${objectName}`, error as Error, { + event, + objectName, + }); + return Promise.resolve(); + } + })); + } + + /** + * Remove all hooks from a package + */ + removePackage(packageName: string): void { + // Remove from all hooks registry + const hooksToRemove: string[] = []; + for (const [hookId, hook] of this.allHooks.entries()) { + if (hook.packageName === packageName) { + hooksToRemove.push(hookId); + } + } + hooksToRemove.forEach(id => this.allHooks.delete(id)); + + // Remove from pipelines + for (const [key, hooks] of this.pipelines.entries()) { + const filtered = hooks.filter(h => h.packageName !== packageName); + if (filtered.length === 0) { + this.pipelines.delete(key); + } else { + this.pipelines.set(key, filtered); + } + } + } + + /** + * Clear all hooks + */ + clear(): void { + this.pipelines.clear(); + this.allHooks.clear(); + } + + /** + * Get statistics about registered hooks + */ + getStats(): { totalHooks: number; totalPipelines: number } { + return { + totalHooks: this.allHooks.size, + totalPipelines: this.pipelines.size + }; + } +} diff --git a/packages/foundation/plugin-optimizations/src/DependencyGraph.ts b/packages/foundation/plugin-optimizations/src/DependencyGraph.ts new file mode 100644 index 00000000..9826149e --- /dev/null +++ b/packages/foundation/plugin-optimizations/src/DependencyGraph.ts @@ -0,0 +1,255 @@ +/** + * ObjectQL + * Copyright (c) 2026-present ObjectStack Inc. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +/** + * Dependency type + */ +export type DependencyType = 'lookup' | 'master_detail' | 'foreign_key'; + +/** + * Edge in the dependency graph + */ +export interface DependencyEdge { + from: string; + to: string; + type: DependencyType; + fieldName: string; +} + +/** + * Smart Dependency Graph + * + * Improvement: DAG-based dependency resolution for cascading operations. + * Automatically handles cascade deletes and updates in correct order. + * + * Expected: Eliminates manual cascade logic, prevents orphaned data + */ +export class DependencyGraph { + // Adjacency list: object -> list of dependent objects + private graph = new Map>(); + + // Store edge metadata + private edges = new Map(); + + /** + * Add an object to the graph + */ + addObject(objectName: string): void { + if (!this.graph.has(objectName)) { + this.graph.set(objectName, new Set()); + } + if (!this.edges.has(objectName)) { + this.edges.set(objectName, []); + } + } + + /** + * Add a dependency edge + * from -> to means "to depends on from" + */ + addDependency(from: string, to: string, type: DependencyType, fieldName: string): void { + this.addObject(from); + this.addObject(to); + + // Add edge + this.graph.get(from)!.add(to); + + // Store edge metadata + const edge: DependencyEdge = { from, to, type, fieldName }; + const fromEdges = this.edges.get(from) || []; + fromEdges.push(edge); + this.edges.set(from, fromEdges); + } + + /** + * Get all objects that depend on the given object + */ + getDependents(objectName: string): string[] { + return Array.from(this.graph.get(objectName) || []); + } + + /** + * Topological sort using DFS + */ + topologicalSort(objects: string[]): string[] { + const visited = new Set(); + const stack: string[] = []; + + const dfs = (node: string) => { + if (visited.has(node)) return; + visited.add(node); + + const dependents = this.graph.get(node); + if (dependents) { + for (const dependent of dependents) { + if (objects.includes(dependent)) { + dfs(dependent); + } + } + } + + stack.push(node); + }; + + for (const obj of objects) { + dfs(obj); + } + + return stack; + } + + /** + * Check for circular dependencies + */ + hasCircularDependency(): boolean { + const visited = new Set(); + const recursionStack = new Set(); + + const hasCycle = (node: string): boolean => { + visited.add(node); + recursionStack.add(node); + + const dependents = this.graph.get(node); + if (dependents) { + for (const dependent of dependents) { + if (!visited.has(dependent)) { + if (hasCycle(dependent)) { + return true; + } + } else if (recursionStack.has(dependent)) { + return true; + } + } + } + + recursionStack.delete(node); + return false; + }; + + for (const node of this.graph.keys()) { + if (!visited.has(node)) { + if (hasCycle(node)) { + return true; + } + } + } + + return false; + } + + /** + * Get cascade delete order for an object + * Returns objects in the order they should be deleted + */ + getCascadeDeleteOrder(objectName: string): string[] { + const dependents = this.getDependents(objectName); + if (dependents.length === 0) { + return [objectName]; + } + + // Recursively get all transitive dependents + const allDependents = new Set(); + const collectDependents = (obj: string) => { + const deps = this.getDependents(obj); + for (const dep of deps) { + if (!allDependents.has(dep)) { + allDependents.add(dep); + collectDependents(dep); + } + } + }; + collectDependents(objectName); + + // Add the original object + allDependents.add(objectName); + + // Sort topologically to get correct deletion order + const sorted = this.topologicalSort(Array.from(allDependents)); + + return sorted; + } + + /** + * Automatically cascade delete based on dependency graph + * + * @param objectName The object type being deleted + * @param id The ID of the record being deleted + * @param deleteFunc Function to delete a record: (objectName, id) => Promise + */ + async cascadeDelete( + objectName: string, + id: string, + deleteFunc: (objName: string, recordId: string) => Promise + ): Promise { + const deleteOrder = this.getCascadeDeleteOrder(objectName); + + // Delete in correct order based on DAG + for (const objToDelete of deleteOrder) { + if (objToDelete === objectName) { + // Delete the main record + await deleteFunc(objectName, id); + } else { + // Find and delete dependent records + // This is a simplified version - in production, you'd need to: + // 1. Query for records that reference the deleted record + // 2. Delete them based on cascade rules (CASCADE vs SET NULL vs RESTRICT) + + const edgesFromParent = this.edges.get(objectName) || []; + for (const edge of edgesFromParent) { + if (edge.to === objToDelete && edge.type === 'master_detail') { + // For master-detail, cascade delete dependent records + // await deleteFunc(objToDelete, ); + // Implementation would require querying for dependent records + } + } + } + } + } + + /** + * Get graph statistics + */ + getStats(): { + totalObjects: number; + totalDependencies: number; + hasCircularDependency: boolean; + } { + let totalDeps = 0; + for (const deps of this.graph.values()) { + totalDeps += deps.size; + } + + return { + totalObjects: this.graph.size, + totalDependencies: totalDeps, + hasCircularDependency: this.hasCircularDependency() + }; + } + + /** + * Clear the graph + */ + clear(): void { + this.graph.clear(); + this.edges.clear(); + } + + /** + * Export graph as DOT format for visualization + */ + toDot(): string { + let dot = 'digraph Dependencies {\n'; + for (const [from, dependents] of this.graph.entries()) { + for (const to of dependents) { + dot += ` "${from}" -> "${to}";\n`; + } + } + dot += '}'; + return dot; + } +} diff --git a/packages/foundation/plugin-optimizations/src/GlobalConnectionPool.ts b/packages/foundation/plugin-optimizations/src/GlobalConnectionPool.ts new file mode 100644 index 00000000..bcea090f --- /dev/null +++ b/packages/foundation/plugin-optimizations/src/GlobalConnectionPool.ts @@ -0,0 +1,253 @@ +/** + * ObjectQL + * Copyright (c) 2026-present ObjectStack Inc. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +import { ObjectQLError } from '@objectql/types'; + +/** + * Connection interface + */ +export interface Connection { + id: string; + driverName: string; + inUse: boolean; + createdAt: number; + lastUsedAt: number; + release: () => Promise; +} + +/** + * Connection pool limits + */ +export interface PoolLimits { + total: number; + perDriver: number; +} + +/** + * Global Connection Pool Manager + * + * Improvement: Kernel-level connection pool with global limits. + * Coordinates connection allocation across all drivers. + * + * Expected: 5x faster connection acquisition, predictable resource usage + */ +export class GlobalConnectionPool { + private limits: PoolLimits; + private allocations = new Map(); + private connections = new Map(); + private waitQueue: Array<{ + driverName: string; + resolve: (conn: Connection) => void; + reject: (error: Error) => void; + }> = []; + + constructor(limits: PoolLimits = { total: 50, perDriver: 20 }) { + this.limits = limits; + } + + /** + * Get total number of active connections across all drivers + */ + private totalConnections(): number { + let total = 0; + for (const count of this.allocations.values()) { + total += count; + } + return total; + } + + /** + * Get number of connections for a specific driver + */ + private getDriverConnections(driverName: string): number { + return this.allocations.get(driverName) || 0; + } + + /** + * Try to process the wait queue + */ + private processWaitQueue(): void { + if (this.waitQueue.length === 0) return; + + // Check if we can fulfill any waiting requests + for (let i = 0; i < this.waitQueue.length; i++) { + const request = this.waitQueue[i]; + + // Check if we can allocate + if (this.canAllocate(request.driverName)) { + this.waitQueue.splice(i, 1); + + // Try to acquire connection + this.doAcquire(request.driverName) + .then(request.resolve) + .catch(request.reject); + + // Only process one at a time to avoid over-allocation + break; + } + } + } + + /** + * Check if we can allocate a connection for a driver + */ + private canAllocate(driverName: string): boolean { + const totalConns = this.totalConnections(); + const driverConns = this.getDriverConnections(driverName); + + // Check if there's an idle connection available + const driverConnections = this.connections.get(driverName) || []; + const hasIdleConnection = driverConnections.some(c => !c.inUse); + + // Can allocate if: + // 1. There's an idle connection (reuse), OR + // 2. We're under the total and per-driver limits (create new) + return hasIdleConnection || (totalConns < this.limits.total && driverConns < this.limits.perDriver); + } + + /** + * Actually acquire a connection (internal) + */ + private async doAcquire(driverName: string): Promise { + // Check for available idle connection first + const driverConnections = this.connections.get(driverName) || []; + const idleConnection = driverConnections.find(c => !c.inUse); + + if (idleConnection) { + idleConnection.inUse = true; + idleConnection.lastUsedAt = Date.now(); + return idleConnection; + } + + // Verify we can create a new connection (double-check to prevent race conditions) + const totalConns = this.totalConnections(); + const driverConns = this.getDriverConnections(driverName); + if (totalConns >= this.limits.total || driverConns >= this.limits.perDriver) { + throw new ObjectQLError({ code: 'DRIVER_CONNECTION_FAILED', message: `Connection pool limit reached for driver: ${driverName}` }); + } + + // Create new connection + const connectionId = `${driverName}-${Date.now()}-${Math.random()}`; + const connection: Connection = { + id: connectionId, + driverName, + inUse: true, + createdAt: Date.now(), + lastUsedAt: Date.now(), + release: async () => { + connection.inUse = false; + connection.lastUsedAt = Date.now(); + + // Process wait queue when connection is released + this.processWaitQueue(); + } + }; + + // Store connection + if (!this.connections.has(driverName)) { + this.connections.set(driverName, []); + } + this.connections.get(driverName)!.push(connection); + + // Update allocation count + this.allocations.set(driverName, this.getDriverConnections(driverName) + 1); + + return connection; + } + + /** + * Acquire a connection from the pool + */ + async acquire(driverName: string): Promise { + // Check global limits before allocation ✅ + if (!this.canAllocate(driverName)) { + // Add to wait queue + return new Promise((resolve, reject) => { + this.waitQueue.push({ driverName, resolve, reject }); + + // Set timeout to prevent indefinite waiting + setTimeout(() => { + const index = this.waitQueue.findIndex( + r => r.driverName === driverName && r.resolve === resolve + ); + if (index >= 0) { + this.waitQueue.splice(index, 1); + reject(new Error(`Connection pool limit reached for driver: ${driverName}`)); + } + }, 30000); // 30 second timeout + }); + } + + return this.doAcquire(driverName); + } + + /** + * Release a connection back to the pool + */ + async release(connection: Connection): Promise { + await connection.release(); + } + + /** + * Close all connections for a driver + */ + async closeDriver(driverName: string): Promise { + const driverConnections = this.connections.get(driverName); + if (driverConnections) { + // Clear all connections + driverConnections.length = 0; + this.connections.delete(driverName); + this.allocations.delete(driverName); + } + + // Process wait queue + this.processWaitQueue(); + } + + /** + * Get pool statistics + */ + getStats(): { + totalConnections: number; + totalLimit: number; + perDriverLimit: number; + driverStats: Record; + waitQueueSize: number; + } { + const driverStats: Record = {}; + + for (const [driverName, connections] of this.connections.entries()) { + const active = connections.filter(c => c.inUse).length; + const idle = connections.filter(c => !c.inUse).length; + driverStats[driverName] = { active, idle }; + } + + return { + totalConnections: this.totalConnections(), + totalLimit: this.limits.total, + perDriverLimit: this.limits.perDriver, + driverStats, + waitQueueSize: this.waitQueue.length + }; + } + + /** + * Update pool limits + */ + updateLimits(limits: Partial): void { + if (limits.total !== undefined) { + this.limits.total = limits.total; + } + if (limits.perDriver !== undefined) { + this.limits.perDriver = limits.perDriver; + } + + // Process wait queue after limits update + this.processWaitQueue(); + } +} diff --git a/packages/foundation/plugin-optimizations/src/LazyMetadataLoader.ts b/packages/foundation/plugin-optimizations/src/LazyMetadataLoader.ts new file mode 100644 index 00000000..cfb4003e --- /dev/null +++ b/packages/foundation/plugin-optimizations/src/LazyMetadataLoader.ts @@ -0,0 +1,180 @@ +/** + * ObjectQL + * Copyright (c) 2026-present ObjectStack Inc. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +/** + * Object metadata definition + */ +export interface ObjectMetadata { + name: string; + label?: string; + fields: Record; + triggers?: any[]; + workflows?: any[]; + permissions?: any[]; + relatedObjects?: string[]; +} + +/** + * Metadata loader function type + */ +export type MetadataLoader = (objectName: string) => Promise; + +/** + * Lazy Metadata Loader with Smart Caching + * + * Improvement: Loads metadata on-demand instead of eagerly at startup. + * Includes predictive preloading for related objects. + * + * Expected: 10x faster startup, 70% lower initial memory + */ +export class LazyMetadataLoader { + private cache = new Map(); + private loaded = new Set(); + private loading = new Map>(); + private preloadScheduled = new Set(); // Track objects with scheduled preloads + private loader: MetadataLoader; + + constructor(loader: MetadataLoader) { + this.loader = loader; + } + + /** + * Load a single object's metadata + */ + private async loadSingle(objectName: string): Promise { + // Check if already loaded + if (this.loaded.has(objectName)) { + const cached = this.cache.get(objectName); + if (cached) return cached; + } + + // Check if currently loading (avoid duplicate loads) + const existingLoad = this.loading.get(objectName); + if (existingLoad) { + return existingLoad; + } + + // Load metadata + const loadPromise = (async () => { + try { + const metadata = await this.loader(objectName); + this.cache.set(objectName, metadata); + this.loaded.add(objectName); + return metadata; + } finally { + this.loading.delete(objectName); + } + })(); + + this.loading.set(objectName, loadPromise); + return loadPromise; + } + + /** + * Predictive preload: load related objects in the background + */ + private predictivePreload(objectName: string): void { + // Avoid redundant preload scheduling for the same object + if (this.preloadScheduled.has(objectName)) { + return; + } + this.preloadScheduled.add(objectName); + + // Run preloading asynchronously after current call stack to avoid blocking + setImmediate(() => { + const metadata = this.cache.get(objectName); + if (!metadata) return; + + // Extract related object names from various sources + const relatedObjects = new Set(); + + // 1. From explicit relatedObjects field + if (metadata.relatedObjects) { + metadata.relatedObjects.forEach(obj => relatedObjects.add(obj)); + } + + // 2. From lookup/master-detail fields + if (metadata.fields) { + for (const field of Object.values(metadata.fields)) { + if (field.type === 'lookup' || field.type === 'master_detail') { + if (field.reference_to) { + relatedObjects.add(field.reference_to); + } + } + } + } + + // Preload related objects asynchronously (don't await) + for (const relatedObject of relatedObjects) { + if (!this.loaded.has(relatedObject) && !this.loading.has(relatedObject)) { + // Fire and forget - preload in background + this.loadSingle(relatedObject).catch(() => { + // Ignore errors in background preloading + }); + } + } + }); + } + + /** + * Get metadata for an object (loads on-demand if not cached) + */ + async get(objectName: string): Promise { + // Load on first access + const metadata = await this.loadSingle(objectName); + + // Trigger predictive preloading for related objects + this.predictivePreload(objectName); + + return metadata; + } + + /** + * Check if metadata is loaded + */ + isLoaded(objectName: string): boolean { + return this.loaded.has(objectName); + } + + /** + * Preload metadata for specific objects + */ + async preload(objectNames: string[]): Promise { + await Promise.all(objectNames.map(name => this.get(name))); + } + + /** + * Clear cache for an object + */ + invalidate(objectName: string): void { + this.cache.delete(objectName); + this.loaded.delete(objectName); + this.preloadScheduled.delete(objectName); + } + + /** + * Clear all cached metadata + */ + clearAll(): void { + this.cache.clear(); + this.loaded.clear(); + this.loading.clear(); + this.preloadScheduled.clear(); + } + + /** + * Get statistics about loaded metadata + */ + getStats(): { loaded: number; cached: number; loading: number } { + return { + loaded: this.loaded.size, + cached: this.cache.size, + loading: this.loading.size + }; + } +} diff --git a/packages/foundation/plugin-optimizations/src/OptimizedMetadataRegistry.ts b/packages/foundation/plugin-optimizations/src/OptimizedMetadataRegistry.ts new file mode 100644 index 00000000..13277d62 --- /dev/null +++ b/packages/foundation/plugin-optimizations/src/OptimizedMetadataRegistry.ts @@ -0,0 +1,132 @@ +/** + * ObjectQL + * Copyright (c) 2026-present ObjectStack Inc. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +/** + * Optimized Metadata Registry with O(k) package uninstall complexity + * + * Improvement: Uses secondary indexes to achieve O(k) complexity for + * unregisterPackage operation (where k is the number of items in the package) + * instead of O(n*m) (where n is types and m is items per type). + */ + +interface MetadataRef { + type: string; + name: string; +} + +export class OptimizedMetadataRegistry { + private items: Record> = {}; + + // Secondary index: package name -> list of metadata references + private packageIndex = new Map>(); + + constructor() {} + + register(type: string, nameOrConfig: any, config?: any) { + if (!this.items[type]) { + this.items[type] = {}; + } + + let name: string; + let item: any; + + if (config) { + name = nameOrConfig; + item = config; + } else { + item = nameOrConfig; + name = item.name || item.id; + } + + if (name) { + this.items[type][name] = item; + + // Update package index + const packageName = item.package || (item as any)._package || (item as any).packageName; + if (packageName) { + if (!this.packageIndex.has(packageName)) { + this.packageIndex.set(packageName, new Set()); + } + this.packageIndex.get(packageName)!.add({ type, name }); + } + } + } + + get(type: string, name: string): T { + const item = this.items[type]?.[name]; + if (item && item.content) { + return item.content; + } + return item; + } + + list(type: string): T[] { + if (!this.items[type]) return []; + return Object.values(this.items[type]).map((item: any) => { + if (item && item.content) { + return item.content; + } + return item; + }); + } + + getTypes(): string[] { + return Object.keys(this.items); + } + + getEntry(type: string, name: string): T { + return this.items[type]?.[name]; + } + + unregister(type: string, name: string) { + const item = this.items[type]?.[name]; + if (item) { + // Update package index + const packageName = item.package || (item as any)._package || (item as any).packageName; + if (packageName) { + const refs = this.packageIndex.get(packageName); + if (refs) { + // Remove this specific reference + for (const ref of refs) { + if (ref.type === type && ref.name === name) { + refs.delete(ref); + break; + } + } + // Clean up empty package entries + if (refs.size === 0) { + this.packageIndex.delete(packageName); + } + } + } + delete this.items[type][name]; + } + } + + /** + * Optimized package unregistration with O(k) complexity + * where k is the number of items in the package. + * + * Previous complexity: O(n*m) - iterate all types and all items + * New complexity: O(k) - direct lookup via secondary index + */ + unregisterPackage(packageName: string) { + // Direct lookup via secondary index ✅ + const refs = this.packageIndex.get(packageName); + if (refs) { + // Delete each item referenced by this package + for (const ref of refs) { + if (this.items[ref.type]?.[ref.name]) { + delete this.items[ref.type][ref.name]; + } + } + // Remove package from index + this.packageIndex.delete(packageName); + } + } +} diff --git a/packages/foundation/plugin-optimizations/src/OptimizedValidationEngine.ts b/packages/foundation/plugin-optimizations/src/OptimizedValidationEngine.ts new file mode 100644 index 00000000..e1ea7d97 --- /dev/null +++ b/packages/foundation/plugin-optimizations/src/OptimizedValidationEngine.ts @@ -0,0 +1,174 @@ +/** + * ObjectQL + * Copyright (c) 2026-present ObjectStack Inc. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +import { ObjectQLError } from '@objectql/types'; + +/** + * Compiled validator function type + */ +export type ValidatorFunction = (data: any) => boolean | { valid: boolean; errors?: string[] }; + +/** + * Validation schema interface + */ +export interface ValidationSchema { + type: string; + required?: boolean; + properties?: Record; + items?: ValidationSchema; + enum?: any[]; + minimum?: number; + maximum?: number; + minLength?: number; + maxLength?: number; + pattern?: string; +} + +/** + * Optimized Validation Engine + * + * Improvement: Compiles validation rules to optimized validators. + * Validators are compiled once and cached for reuse. + * + * Expected: 3x faster validation, lower memory churn + */ +export class OptimizedValidationEngine { + private validators = new Map(); + + /** + * Compile a validation schema to an optimized validator function + */ + private compileSchema(schema: ValidationSchema): ValidatorFunction { + // Generate optimized validation function + return (data: any): { valid: boolean; errors?: string[] } => { + const errors: string[] = []; + + // Type validation + if (schema.type) { + const actualType = Array.isArray(data) ? 'array' : typeof data; + if (actualType !== schema.type && !(schema.type === 'integer' && typeof data === 'number')) { + errors.push(`Expected type ${schema.type}, got ${actualType}`); + } + } + + // Required validation + if (schema.required && (data === null || data === undefined)) { + errors.push('Value is required'); + } + + // String validations + if (typeof data === 'string') { + if (schema.minLength !== undefined && data.length < schema.minLength) { + errors.push(`String length must be at least ${schema.minLength}`); + } + if (schema.maxLength !== undefined && data.length > schema.maxLength) { + errors.push(`String length must not exceed ${schema.maxLength}`); + } + if (schema.pattern) { + const regex = new RegExp(schema.pattern); + if (!regex.test(data)) { + errors.push(`String does not match pattern ${schema.pattern}`); + } + } + } + + // Number validations + if (typeof data === 'number') { + if (schema.minimum !== undefined && data < schema.minimum) { + errors.push(`Value must be at least ${schema.minimum}`); + } + if (schema.maximum !== undefined && data > schema.maximum) { + errors.push(`Value must not exceed ${schema.maximum}`); + } + } + + // Enum validation + if (schema.enum && !schema.enum.includes(data)) { + errors.push(`Value must be one of: ${schema.enum.join(', ')}`); + } + + // Object property validation + if (schema.properties && typeof data === 'object' && data !== null) { + for (const [key, propSchema] of Object.entries(schema.properties)) { + const propValidator = this.compileSchema(propSchema); + const result = propValidator(data[key]); + if (typeof result === 'object' && !result.valid) { + errors.push(...(result.errors || []).map(e => `${key}: ${e}`)); + } + } + } + + // Array item validation + if (schema.items && Array.isArray(data)) { + const itemValidator = this.compileSchema(schema.items); + data.forEach((item, index) => { + const result = itemValidator(item); + if (typeof result === 'object' && !result.valid) { + errors.push(...(result.errors || []).map(e => `[${index}]: ${e}`)); + } + }); + } + + return { + valid: errors.length === 0, + errors: errors.length > 0 ? errors : undefined + }; + }; + } + + /** + * Compile and cache a validator for an object + */ + compile(objectName: string, schema: ValidationSchema): void { + const validator = this.compileSchema(schema); + this.validators.set(objectName, validator); + } + + /** + * Validate data against a compiled validator + */ + validate(objectName: string, data: any): { valid: boolean; errors?: string[] } { + const validator = this.validators.get(objectName); + if (!validator) { + throw new ObjectQLError({ code: 'VALIDATION_ERROR', message: `No validator compiled for object: ${objectName}` }); + } + + const result = validator(data); + return typeof result === 'boolean' ? { valid: result } : result; + } + + /** + * Check if a validator exists for an object + */ + hasValidator(objectName: string): boolean { + return this.validators.has(objectName); + } + + /** + * Remove a compiled validator + */ + removeValidator(objectName: string): void { + this.validators.delete(objectName); + } + + /** + * Clear all compiled validators + */ + clearAll(): void { + this.validators.clear(); + } + + /** + * Get statistics about compiled validators + */ + getStats(): { totalValidators: number } { + return { + totalValidators: this.validators.size + }; + } +} diff --git a/packages/foundation/plugin-optimizations/src/QueryCompiler.ts b/packages/foundation/plugin-optimizations/src/QueryCompiler.ts new file mode 100644 index 00000000..6047acb5 --- /dev/null +++ b/packages/foundation/plugin-optimizations/src/QueryCompiler.ts @@ -0,0 +1,242 @@ +/** + * ObjectQL + * Copyright (c) 2026-present ObjectStack Inc. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +/** + * LRU Cache implementation + * Simple doubly-linked list + hash map for O(1) operations + */ +class LRUCache { + private capacity: number; + private cache = new Map(); + private head: K | null = null; + private tail: K | null = null; + + constructor(capacity: number) { + this.capacity = capacity; + } + + get(key: K): V | undefined { + const node = this.cache.get(key); + if (!node) return undefined; + + // Move to front (most recently used) + this.moveToFront(key); + return node.value; + } + + set(key: K, value: V): void { + if (this.cache.has(key)) { + // Update existing + const node = this.cache.get(key)!; + node.value = value; + this.moveToFront(key); + } else { + // Add new + if (this.cache.size >= this.capacity) { + // Evict least recently used (tail) + if (this.tail !== null) { + const oldTail = this.tail; + const tailNode = this.cache.get(this.tail); + if (tailNode && tailNode.prev !== null) { + const prevNode = this.cache.get(tailNode.prev); + if (prevNode) { + prevNode.next = null; + this.tail = tailNode.prev; + } + } else { + this.head = null; + this.tail = null; + } + this.cache.delete(oldTail); + } + } + + // Insert at head + this.cache.set(key, { value, prev: null, next: this.head }); + if (this.head !== null) { + const headNode = this.cache.get(this.head); + if (headNode) { + headNode.prev = key; + } + } + this.head = key; + if (this.tail === null) { + this.tail = key; + } + } + } + + has(key: K): boolean { + return this.cache.has(key); + } + + private moveToFront(key: K): void { + if (key === this.head) return; // Already at front + + const node = this.cache.get(key); + if (!node) return; + + // Remove from current position + if (node.prev !== null) { + const prevNode = this.cache.get(node.prev); + if (prevNode) { + prevNode.next = node.next; + } + } + if (node.next !== null) { + const nextNode = this.cache.get(node.next); + if (nextNode) { + nextNode.prev = node.prev; + } + } + if (key === this.tail) { + this.tail = node.prev; + } + + // Move to front + node.prev = null; + node.next = this.head; + if (this.head !== null) { + const headNode = this.cache.get(this.head); + if (headNode) { + headNode.prev = key; + } + } + this.head = key; + } +} + +/** + * Compiled Query representation + * Contains optimized execution plan + */ +export interface CompiledQuery { + objectName: string; + ast: any; + plan: any; + timestamp: number; +} + +/** + * Query Compiler with LRU Cache + * + * Improvement: Compiles Query AST to optimized execution plan and caches results. + * Expected: 10x faster query planning, 50% lower CPU usage + */ +export class QueryCompiler { + private cache: LRUCache; + + constructor(cacheSize: number = 1000) { + this.cache = new LRUCache(cacheSize); + } + + /** + * Hash a Query AST to create a cache key + */ + private hashAST(ast: any): string { + // Simple JSON-based hash for now + // In production, consider a faster hash function + try { + return JSON.stringify(ast); + } catch (_e) { + // Fallback for circular references + return String(Date.now() + Math.random()); + } + } + + /** + * Compile AST to optimized execution plan + */ + private compileAST(objectName: string, ast: any): CompiledQuery { + // Optimization opportunities: + // 1. Precompute field projections + // 2. Optimize filter conditions + // 3. Determine optimal join strategy + // 4. Index hint detection + + const plan = { + objectName, + // Extract and optimize components + fields: ast.fields || ['*'], + filters: ast.filters || ast.where, + sort: ast.sort || ast.orderBy, + limit: ast.limit || ast.top, + offset: ast.offset || ast.skip, + // Add optimization hints + useIndex: this.detectIndexableFields(ast), + joinStrategy: this.determineJoinStrategy(ast) + }; + + return { + objectName, + ast, + plan, + timestamp: Date.now() + }; + } + + /** + * Detect fields that can use indexes + */ + private detectIndexableFields(ast: any): string[] { + const indexable: string[] = []; + + if (ast.filters) { + // Extract fields from filter conditions + const extractFields = (filters: any): void => { + if (Array.isArray(filters)) { + filters.forEach(extractFields); + } else if (filters && typeof filters === 'object') { + Object.keys(filters).forEach(key => { + if (!key.startsWith('$')) { + indexable.push(key); + } + }); + } + }; + extractFields(ast.filters); + } + + return [...new Set(indexable)]; // Remove duplicates + } + + /** + * Determine optimal join strategy + */ + private determineJoinStrategy(ast: any): 'nested' | 'hash' | 'merge' { + // Simple heuristic: use hash join for large datasets + if (ast.limit && ast.limit < 100) { + return 'nested'; + } + return 'hash'; + } + + /** + * Compile and cache query + */ + compile(objectName: string, ast: any): CompiledQuery { + const key = this.hashAST(ast); + + if (this.cache.has(key)) { + // Cache hit ✅ + return this.cache.get(key)!; + } + + // Cache miss - compile and store + const compiled = this.compileAST(objectName, ast); + this.cache.set(key, compiled); + return compiled; + } + + /** + * Clear the cache (useful for testing or after schema changes) + */ + clearCache(): void { + this.cache = new LRUCache(1000); + } +} diff --git a/packages/foundation/plugin-optimizations/src/SQLQueryOptimizer.ts b/packages/foundation/plugin-optimizations/src/SQLQueryOptimizer.ts new file mode 100644 index 00000000..f0173c7a --- /dev/null +++ b/packages/foundation/plugin-optimizations/src/SQLQueryOptimizer.ts @@ -0,0 +1,330 @@ +/** + * ObjectQL + * Copyright (c) 2026-present ObjectStack Inc. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +/** + * Index metadata + */ +export interface IndexMetadata { + name: string; + fields: string[]; + unique: boolean; + type?: 'btree' | 'hash' | 'fulltext'; +} + +/** + * Schema with index information + */ +export interface SchemaWithIndexes { + name: string; + fields: Record; + indexes?: IndexMetadata[]; +} + +/** + * Query AST for optimization + */ +export interface OptimizableQueryAST { + object: string; + fields?: string[]; + filters?: any; + sort?: Array<{ field: string; order: 'asc' | 'desc' }>; + joins?: Array<{ type: 'left' | 'inner'; table: string; on: any }>; + limit?: number; + offset?: number; +} + +/** + * SQL Query Optimizer + * + * Improvement: SQL-aware optimization with index hints and join reordering. + * Analyzes query patterns and schema to generate optimal SQL. + * + * Expected: 2-5x faster queries on large datasets + */ +export class SQLQueryOptimizer { + private schemas = new Map(); + + /** + * Register a schema with index information + */ + registerSchema(schema: SchemaWithIndexes): void { + this.schemas.set(schema.name, schema); + } + + /** + * Check if a field has an index + */ + private hasIndex(objectName: string, fieldName: string): boolean { + const schema = this.schemas.get(objectName); + if (!schema || !schema.indexes) return false; + + return schema.indexes.some(index => + index.fields.includes(fieldName) + ); + } + + /** + * Get the best index for given filter fields + */ + private getBestIndex(objectName: string, filterFields: string[]): IndexMetadata | null { + const schema = this.schemas.get(objectName); + if (!schema || !schema.indexes) return null; + + // Find indexes that cover the filter fields + const candidateIndexes = schema.indexes.filter(index => { + // Check if index covers any of the filter fields + return filterFields.some(field => index.fields.includes(field)); + }); + + if (candidateIndexes.length === 0) return null; + + // Prefer indexes that match more filter fields + candidateIndexes.sort((a, b) => { + const aMatches = a.fields.filter(f => filterFields.includes(f)).length; + const bMatches = b.fields.filter(f => filterFields.includes(f)).length; + return bMatches - aMatches; + }); + + return candidateIndexes[0]; + } + + /** + * Extract filter fields from filter conditions + */ + private extractFilterFields(filters: any): string[] { + const fields: string[] = []; + + const extract = (obj: any) => { + if (!obj || typeof obj !== 'object') return; + + for (const key of Object.keys(obj)) { + if (!key.startsWith('$')) { + fields.push(key); + } + if (typeof obj[key] === 'object') { + extract(obj[key]); + } + } + }; + + extract(filters); + return [...new Set(fields)]; // Remove duplicates + } + + /** + * Optimize join type based on query characteristics + * + * Note: This method expects filter fields in one of two formats: + * 1. Table-prefixed: "tableName.fieldName" (e.g., "accounts.type") + * 2. Simple field name: "fieldName" (checked against joined table schema) + * + * Multi-level qualification (e.g., "schema.table.field") is not currently supported. + */ + private optimizeJoinType( + join: { type: 'left' | 'inner'; table: string; on: any }, + ast: OptimizableQueryAST + ): 'left' | 'inner' { + // If we're filtering on the joined table, we can use INNER JOIN + if (ast.filters) { + const filterFields = this.extractFilterFields(ast.filters); + + // Check if any filter field references the joined table + const hasFilterOnJoinTable = filterFields.some(field => { + // Handle table-prefixed fields like "accounts.type" + if (field.includes('.')) { + const parts = field.split('.'); + // Only consider the first part as table name for simplicity + // This assumes format: "table.field" not "schema.table.field" + if (parts.length === 2 && parts[0] === join.table) { + return true; + } + // If more than 2 parts or table doesn't match, skip this field + return false; + } + + // Also check if the field exists in the joined table schema (non-prefixed) + const schema = this.schemas.get(join.table); + if (schema) { + return schema.fields[field] !== undefined; + } + + return false; + }); + + if (hasFilterOnJoinTable) { + return 'inner'; // Convert LEFT to INNER for better performance + } + } + + return join.type; + } + + /** + * Optimize a query AST to SQL + */ + optimize(ast: OptimizableQueryAST): string { + const schema = this.schemas.get(ast.object); + if (!schema) { + // Fallback to basic SQL generation + return this.generateBasicSQL(ast); + } + + let sql = 'SELECT '; + + // Fields + if (ast.fields && ast.fields.length > 0) { + sql += ast.fields.join(', '); + } else { + sql += '*'; + } + + sql += ` FROM ${ast.object}`; + + // Index hints + if (ast.filters) { + const filterFields = this.extractFilterFields(ast.filters); + const bestIndex = this.getBestIndex(ast.object, filterFields); + + if (bestIndex) { + // Add index hint for MySQL/MariaDB + sql += ` USE INDEX (${bestIndex.name})`; + } + } + + // Optimized joins + if (ast.joins && ast.joins.length > 0) { + for (const join of ast.joins) { + const optimizedType = this.optimizeJoinType(join, ast); + sql += ` ${optimizedType.toUpperCase()} JOIN ${join.table}`; + + // Simplified join condition + if (typeof join.on === 'string') { + sql += ` ON ${join.on}`; + } + } + } + + // WHERE clause + if (ast.filters) { + sql += ' WHERE ' + this.filtersToSQL(ast.filters); + } + + // ORDER BY + if (ast.sort && ast.sort.length > 0) { + sql += ' ORDER BY '; + sql += ast.sort.map(s => `${s.field} ${s.order.toUpperCase()}`).join(', '); + } + + // LIMIT and OFFSET + if (ast.limit !== undefined) { + sql += ` LIMIT ${ast.limit}`; + } + if (ast.offset !== undefined) { + sql += ` OFFSET ${ast.offset}`; + } + + return sql; + } + + /** + * Convert filter object to SQL WHERE clause + */ + private filtersToSQL(filters: any): string { + if (typeof filters === 'string') { + return filters; + } + + // Simplified filter conversion + const conditions: string[] = []; + + for (const [key, value] of Object.entries(filters)) { + if (key === '$and') { + const subconditions = (value as any[]).map(f => this.filtersToSQL(f)); + conditions.push(`(${subconditions.join(' AND ')})`); + } else if (key === '$or') { + const subconditions = (value as any[]).map(f => this.filtersToSQL(f)); + conditions.push(`(${subconditions.join(' OR ')})`); + } else if (!key.startsWith('$')) { + // Field condition + if (typeof value === 'object' && value !== null) { + for (const [op, val] of Object.entries(value)) { + switch (op) { + case '$eq': + conditions.push(`${key} = '${val}'`); + break; + case '$ne': + conditions.push(`${key} != '${val}'`); + break; + case '$gt': + conditions.push(`${key} > '${val}'`); + break; + case '$gte': + conditions.push(`${key} >= '${val}'`); + break; + case '$lt': + conditions.push(`${key} < '${val}'`); + break; + case '$lte': + conditions.push(`${key} <= '${val}'`); + break; + case '$in': { + const inValues = (val as any[]).map(v => `'${v}'`).join(', '); + conditions.push(`${key} IN (${inValues})`); + break; + } + } + } + } else { + conditions.push(`${key} = '${value}'`); + } + } + } + + return conditions.join(' AND '); + } + + /** + * Fallback: generate basic SQL without optimizations + */ + private generateBasicSQL(ast: OptimizableQueryAST): string { + let sql = 'SELECT '; + + if (ast.fields && ast.fields.length > 0) { + sql += ast.fields.join(', '); + } else { + sql += '*'; + } + + sql += ` FROM ${ast.object}`; + + if (ast.filters) { + sql += ' WHERE ' + this.filtersToSQL(ast.filters); + } + + if (ast.sort && ast.sort.length > 0) { + sql += ' ORDER BY '; + sql += ast.sort.map(s => `${s.field} ${s.order.toUpperCase()}`).join(', '); + } + + if (ast.limit !== undefined) { + sql += ` LIMIT ${ast.limit}`; + } + if (ast.offset !== undefined) { + sql += ` OFFSET ${ast.offset}`; + } + + return sql; + } + + /** + * Clear all registered schemas + */ + clearSchemas(): void { + this.schemas.clear(); + } +} diff --git a/packages/foundation/plugin-optimizations/src/index.ts b/packages/foundation/plugin-optimizations/src/index.ts new file mode 100644 index 00000000..9f7aec82 --- /dev/null +++ b/packages/foundation/plugin-optimizations/src/index.ts @@ -0,0 +1,32 @@ +/** + * @objectql/plugin-optimizations + * Copyright (c) 2026-present ObjectStack Inc. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +/** + * Kernel Optimizations Plugin + * + * This plugin provides performance optimizations for the ObjectQL kernel: + * + * 1. OptimizedMetadataRegistry - O(k) package uninstall with secondary indexes + * 2. QueryCompiler - LRU cache for compiled query plans + * 3. CompiledHookManager - Pre-compiled hook pipelines + * 4. GlobalConnectionPool - Kernel-level connection pooling + * 5. OptimizedValidationEngine - Compiled validation rules + * 6. LazyMetadataLoader - On-demand metadata loading with predictive preload + * 7. DependencyGraph - DAG-based dependency resolution + * 8. SQLQueryOptimizer - SQL-aware query optimization with index hints + */ + +export { OptimizationsPlugin } from './plugin'; +export { OptimizedMetadataRegistry } from './OptimizedMetadataRegistry'; +export { QueryCompiler, type CompiledQuery } from './QueryCompiler'; +export { CompiledHookManager, type Hook } from './CompiledHookManager'; +export { GlobalConnectionPool, type Connection, type PoolLimits } from './GlobalConnectionPool'; +export { OptimizedValidationEngine, type ValidatorFunction, type ValidationSchema } from './OptimizedValidationEngine'; +export { LazyMetadataLoader, type ObjectMetadata, type MetadataLoader } from './LazyMetadataLoader'; +export { DependencyGraph, type DependencyEdge, type DependencyType } from './DependencyGraph'; +export { SQLQueryOptimizer, type IndexMetadata, type SchemaWithIndexes, type OptimizableQueryAST } from './SQLQueryOptimizer'; diff --git a/packages/foundation/plugin-optimizations/src/plugin.ts b/packages/foundation/plugin-optimizations/src/plugin.ts new file mode 100644 index 00000000..b873c841 --- /dev/null +++ b/packages/foundation/plugin-optimizations/src/plugin.ts @@ -0,0 +1,124 @@ +/** + * @objectql/plugin-optimizations + * Copyright (c) 2026-present ObjectStack Inc. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +import type { RuntimePlugin, RuntimeContext } from '@objectql/types'; +import { ConsoleLogger } from '@objectql/types'; +import type { Logger } from '@objectql/types'; +import { OptimizedMetadataRegistry } from './OptimizedMetadataRegistry'; +import { GlobalConnectionPool } from './GlobalConnectionPool'; +import { QueryCompiler } from './QueryCompiler'; + +/** + * Configuration for the Optimizations Plugin + */ +export interface OptimizationsPluginConfig { + /** + * Enable optimized metadata registry + * @default true + */ + enableOptimizedRegistry?: boolean; + + /** + * Enable global connection pooling + * @default true + */ + enableConnectionPool?: boolean; + + /** + * Connection pool limits + */ + poolLimits?: { total?: number; perDriver?: number }; + + /** + * Enable query compiler with LRU cache + * @default true + */ + enableQueryCompiler?: boolean; + + /** + * Query compiler cache size + * @default 1000 + */ + queryCompilerCacheSize?: number; +} + +/** + * Optimizations Plugin + * + * Provides performance optimizations for the ObjectQL kernel: + * - Optimized metadata registry with O(k) package uninstall + * - Global connection pooling across drivers + * - Query compiler with LRU cache + */ +export class OptimizationsPlugin implements RuntimePlugin { + name = '@objectql/plugin-optimizations'; + version = '4.2.0'; + private logger: Logger; + private config: Required; + + constructor(config: OptimizationsPluginConfig = {}) { + this.config = { + enableOptimizedRegistry: true, + enableConnectionPool: true, + poolLimits: { total: 50, perDriver: 20 }, + enableQueryCompiler: true, + queryCompilerCacheSize: 1000, + ...config, + }; + this.logger = new ConsoleLogger({ name: this.name, level: 'info' }); + } + + async install(ctx: RuntimeContext): Promise { + this.logger.info('Installing optimizations plugin...'); + + const kernel = ctx.engine as any; + + // 1. Wrap metadata registry with optimized version + if (this.config.enableOptimizedRegistry && kernel.metadata) { + const existingRegistry = kernel.metadata; + const optimized = new OptimizedMetadataRegistry(); + + // Copy existing items to optimized registry + if (typeof existingRegistry.getTypes === 'function') { + for (const type of existingRegistry.getTypes()) { + const items = existingRegistry.list(type); + for (const item of items) { + optimized.register(type, item); + } + } + } + + // Replace metadata on kernel if replaceService is available + if (typeof (ctx as any).replaceService === 'function') { + (ctx as any).replaceService('metadata', optimized); + } + kernel.optimizedMetadata = optimized; + this.logger.debug('Optimized metadata registry installed'); + } + + // 2. Initialize connection pooling + if (this.config.enableConnectionPool) { + const connectionPool = new GlobalConnectionPool(this.config.poolLimits as any); + kernel.connectionPool = connectionPool; + this.logger.debug('Global connection pool installed'); + } + + // 3. Initialize query compiler + if (this.config.enableQueryCompiler) { + const queryCompiler = new QueryCompiler(this.config.queryCompilerCacheSize); + kernel.queryCompiler = queryCompiler; + this.logger.debug('Query compiler installed'); + } + + this.logger.info('Optimizations plugin installed successfully'); + } + + async onStart(_ctx: RuntimeContext): Promise { + this.logger.debug('Optimizations plugin started'); + } +} diff --git a/packages/foundation/plugin-optimizations/tsconfig.json b/packages/foundation/plugin-optimizations/tsconfig.json new file mode 100644 index 00000000..588ec401 --- /dev/null +++ b/packages/foundation/plugin-optimizations/tsconfig.json @@ -0,0 +1,12 @@ +{ + "extends": "../../../tsconfig.base.json", + "compilerOptions": { + "outDir": "dist", + "rootDir": "src" + }, + "include": ["src/**/*"], + "exclude": ["node_modules", "dist"], + "references": [ + { "path": "../types" } + ] +} diff --git a/packages/foundation/plugin-query/package.json b/packages/foundation/plugin-query/package.json new file mode 100644 index 00000000..915b7636 --- /dev/null +++ b/packages/foundation/plugin-query/package.json @@ -0,0 +1,28 @@ +{ + "name": "@objectql/plugin-query", + "version": "4.2.0", + "description": "Query execution and analysis plugin for ObjectQL - query service, builder, and analyzer", + "main": "dist/index.js", + "types": "dist/index.d.ts", + "sideEffects": false, + "exports": { + ".": { + "types": "./dist/index.d.ts", + "default": "./dist/index.js" + } + }, + "files": ["dist"], + "scripts": { + "build": "tsc", + "test": "vitest run" + }, + "dependencies": { + "@objectql/types": "workspace:*", + "@objectql/plugin-optimizations": "workspace:*", + "@objectstack/core": "^2.0.6", + "@objectstack/spec": "^2.0.6" + }, + "devDependencies": { + "typescript": "^5.3.0" + } +} diff --git a/packages/foundation/plugin-query/src/filter-translator.ts b/packages/foundation/plugin-query/src/filter-translator.ts new file mode 100644 index 00000000..d758541f --- /dev/null +++ b/packages/foundation/plugin-query/src/filter-translator.ts @@ -0,0 +1,38 @@ +/** + * ObjectQL + * Copyright (c) 2026-present ObjectStack Inc. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +import type { Filter } from '@objectql/types'; + +/** + * Filter Translator + * + * Translates ObjectQL Filter to ObjectStack FilterCondition format. + * Since both now use the same format, this is mostly a pass-through. + * + * @example + * Input: { age: { $gte: 18 }, $or: [{ status: "active" }, { role: "admin" }] } + * Output: { age: { $gte: 18 }, $or: [{ status: "active" }, { role: "admin" }] } + */ +export class FilterTranslator { + /** + * Translate filters from ObjectQL format to ObjectStack FilterCondition format + */ + translate(filters?: Filter): Filter | undefined { + if (!filters) { + return undefined; + } + + // If it's an empty object, return undefined + if (typeof filters === 'object' && Object.keys(filters).length === 0) { + return undefined; + } + + // Both ObjectQL Filter and ObjectStack FilterCondition use the same format now + return filters; + } +} diff --git a/packages/foundation/plugin-query/src/index.ts b/packages/foundation/plugin-query/src/index.ts new file mode 100644 index 00000000..00f24203 --- /dev/null +++ b/packages/foundation/plugin-query/src/index.ts @@ -0,0 +1,22 @@ +/** + * @objectql/plugin-query + * Copyright (c) 2026-present ObjectStack Inc. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +/** + * Query Plugin + * + * Provides query execution and analysis capabilities: + * - QueryService — query execution with profiling support + * - QueryBuilder — builds ObjectStack QueryAST from ObjectQL UnifiedQuery + * - QueryAnalyzer — query performance analysis and optimization suggestions + */ + +export { QueryPlugin } from './plugin'; +export { QueryService, type QueryOptions, type QueryResult, type QueryProfile } from './query-service'; +export { QueryBuilder } from './query-builder'; +export { QueryAnalyzer, type QueryPlan, type ProfileResult, type QueryStats } from './query-analyzer'; +export { FilterTranslator } from './filter-translator'; diff --git a/packages/foundation/plugin-query/src/plugin.ts b/packages/foundation/plugin-query/src/plugin.ts new file mode 100644 index 00000000..e0603bc2 --- /dev/null +++ b/packages/foundation/plugin-query/src/plugin.ts @@ -0,0 +1,91 @@ +/** + * @objectql/plugin-query + * Copyright (c) 2026-present ObjectStack Inc. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +import type { RuntimePlugin, RuntimeContext } from '@objectql/types'; +import { ConsoleLogger } from '@objectql/types'; +import type { Logger, Driver } from '@objectql/types'; +import { QueryService } from './query-service'; +import { QueryAnalyzer } from './query-analyzer'; + +/** + * Configuration for the Query Plugin + */ +export interface QueryPluginConfig { + /** + * Datasources for query service + */ + datasources?: Record; + + /** + * Enable query analyzer + * @default true + */ + enableAnalyzer?: boolean; +} + +/** + * Query Plugin + * + * Provides query execution and analysis capabilities for the ObjectQL kernel. + * Registers QueryService and QueryAnalyzer on the kernel for consumer access. + */ +export class QueryPlugin implements RuntimePlugin { + name = '@objectql/plugin-query'; + version = '4.2.0'; + private logger: Logger; + + constructor(private config: QueryPluginConfig = {}) { + this.config = { + enableAnalyzer: true, + ...config, + }; + this.logger = new ConsoleLogger({ name: this.name, level: 'info' }); + } + + async install(ctx: RuntimeContext): Promise { + this.logger.info('Installing query plugin...'); + + const kernel = ctx.engine as any; + + // Get datasources - either from config or from kernel drivers + let datasources = this.config.datasources; + if (!datasources) { + const drivers = kernel.getAllDrivers?.(); + if (drivers && drivers.length > 0) { + datasources = {}; + drivers.forEach((driver: any, index: number) => { + const driverName = driver.name || (index === 0 ? 'default' : `driver_${index + 1}`); + datasources![driverName] = driver; + }); + } + } + + if (!datasources) { + this.logger.warn('No datasources available. QueryService will not be registered.'); + return; + } + + // Create QueryService + const queryService = new QueryService(datasources, kernel.metadata); + kernel.queryService = queryService; + this.logger.debug('QueryService registered'); + + // Create QueryAnalyzer if enabled + if (this.config.enableAnalyzer !== false) { + const queryAnalyzer = new QueryAnalyzer(queryService, kernel.metadata); + kernel.queryAnalyzer = queryAnalyzer; + this.logger.debug('QueryAnalyzer registered'); + } + + this.logger.info('Query plugin installed successfully'); + } + + async onStart(_ctx: RuntimeContext): Promise { + this.logger.debug('Query plugin started'); + } +} diff --git a/packages/foundation/plugin-query/src/query-analyzer.ts b/packages/foundation/plugin-query/src/query-analyzer.ts new file mode 100644 index 00000000..d2dcb574 --- /dev/null +++ b/packages/foundation/plugin-query/src/query-analyzer.ts @@ -0,0 +1,532 @@ +/** + * ObjectQL Query Analyzer + * Copyright (c) 2026-present ObjectStack Inc. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +import type { UnifiedQuery, ObjectConfig, MetadataRegistry } from '@objectql/types'; +import { QueryAST, ObjectQLError } from '@objectql/types'; +import { QueryService, QueryOptions } from './query-service'; + +/** + * Query execution plan + */ +export interface QueryPlan { + /** + * The compiled QueryAST + */ + ast: QueryAST; + + /** + * Estimated number of rows to be scanned + */ + estimatedRows?: number; + + /** + * Indexes that could be used for this query + */ + indexes: string[]; + + /** + * Warnings about potential performance issues + */ + warnings: string[]; + + /** + * Suggestions for optimization + */ + suggestions: string[]; + + /** + * Complexity score (0-100, higher is more complex) + */ + complexity: number; +} + +/** + * Query profile result with execution statistics + */ +export interface ProfileResult { + /** + * Execution time in milliseconds + */ + executionTime: number; + + /** + * Number of rows scanned by the database + */ + rowsScanned: number; + + /** + * Number of rows returned + */ + rowsReturned: number; + + /** + * Whether an index was used + */ + indexUsed: boolean; + + /** + * The query plan + */ + plan: QueryPlan; + + /** + * Efficiency ratio (rowsReturned / rowsScanned) + * Higher is better (1.0 is perfect, 0.0 is worst) + */ + efficiency: number; +} + +/** + * Aggregated query statistics + */ +export interface QueryStats { + /** + * Total number of queries executed + */ + totalQueries: number; + + /** + * Average execution time in milliseconds + */ + avgExecutionTime: number; + + /** + * Slowest query execution time + */ + slowestQuery: number; + + /** + * Fastest query execution time + */ + fastestQuery: number; + + /** + * Queries by object + */ + byObject: Record; + + /** + * Top slow queries + */ + slowQueries: Array<{ + objectName: string; + executionTime: number; + query: UnifiedQuery; + timestamp: Date; + }>; +} + +/** + * Query Analyzer + * + * Provides query performance analysis and profiling capabilities. + * This class helps developers optimize queries by: + * - Analyzing query plans + * - Profiling execution performance + * - Tracking statistics + * - Providing optimization suggestions + */ +export class QueryAnalyzer { + private stats: QueryStats = { + totalQueries: 0, + avgExecutionTime: 0, + slowestQuery: 0, + fastestQuery: Number.MAX_VALUE, + byObject: {}, + slowQueries: [] + }; + + private executionTimes: number[] = []; + private readonly MAX_SLOW_QUERIES = 10; + + constructor( + private queryService: QueryService, + private metadata: MetadataRegistry + ) {} + + /** + * Analyze a query and generate an execution plan + * + * @param objectName - The object to query + * @param query - The unified query + * @returns Query plan with optimization suggestions + */ + async explain(objectName: string, query: UnifiedQuery): Promise { + const schema = this.getSchema(objectName); + + // Build the QueryAST (without executing) + const ast: QueryAST = { + object: objectName, + where: query.where as any, // FilterCondition format + orderBy: query.orderBy as any, // Will be converted to SortNode[] format + limit: query.limit, + offset: query.offset, + fields: query.fields + }; + + // Analyze filters for index usage + const indexes = this.findApplicableIndexes(schema, query); + + // Detect potential issues + const warnings = this.analyzeWarnings(schema, query); + + // Generate suggestions + const suggestions = this.generateSuggestions(schema, query, indexes); + + // Calculate complexity + const complexity = this.calculateComplexity(query); + + // Try to estimate rows (basic heuristic) + const estimatedRows = this.estimateRows(schema, query); + + return { + ast, + estimatedRows, + indexes, + warnings, + suggestions, + complexity + }; + } + + /** + * Profile a query execution + * + * @param objectName - The object to query + * @param query - The unified query + * @param options - Query execution options + * @returns Profile result with execution statistics + */ + async profile( + objectName: string, + query: UnifiedQuery, + options: QueryOptions = {} + ): Promise { + // Get the query plan first + const plan = await this.explain(objectName, query); + + // Execute with profiling enabled + const result = await this.queryService.find(objectName, query, { + ...options, + profile: true + }); + + const executionTime = result.profile?.executionTime || 0; + const rowsReturned = result.value.length; + const rowsScanned = result.profile?.rowsScanned || rowsReturned; + + // Calculate efficiency + const efficiency = rowsScanned > 0 ? rowsReturned / rowsScanned : 0; + + // Determine if index was used (heuristic) + const indexUsed = plan.indexes.length > 0 && efficiency > 0.5; + + // Update statistics + this.recordExecution(objectName, executionTime, query); + + return { + executionTime, + rowsScanned, + rowsReturned, + indexUsed, + plan, + efficiency + }; + } + + /** + * Get aggregated query statistics + * + * @returns Current query statistics + */ + getStatistics(): QueryStats { + return { ...this.stats }; + } + + /** + * Reset statistics + */ + resetStatistics(): void { + this.stats = { + totalQueries: 0, + avgExecutionTime: 0, + slowestQuery: 0, + fastestQuery: Number.MAX_VALUE, + byObject: {}, + slowQueries: [] + }; + this.executionTimes = []; + } + + /** + * Get schema for an object + * @private + */ + private getSchema(objectName: string): ObjectConfig { + const obj = this.metadata.get('object', objectName); + if (!obj) { + throw new ObjectQLError({ code: 'NOT_FOUND', message: `Object '${objectName}' not found` }); + } + return obj; + } + + /** + * Find indexes that could be used for the query + * @private + */ + private findApplicableIndexes(schema: ObjectConfig, query: UnifiedQuery): string[] { + const indexes: string[] = []; + + if (!schema.indexes || !query.where) { + return indexes; + } + + // Extract fields used in filters + const filterFields = new Set(); + + // FilterCondition is an object-based filter (e.g., { field: value } or { field: { $eq: value } }) + // We need to extract field names from the filter object + const extractFieldsFromFilter = (filter: any): void => { + if (!filter || typeof filter !== 'object') return; + + for (const key of Object.keys(filter)) { + // Skip logical operators + if (key.startsWith('$')) { + // Logical operators contain nested filters + if (key === '$and' || key === '$or') { + const nested = filter[key]; + if (Array.isArray(nested)) { + nested.forEach(extractFieldsFromFilter); + } + } + continue; + } + // This is a field name + filterFields.add(key); + } + }; + + extractFieldsFromFilter(query.where); + + // Check which indexes could be used + const indexesArray = Array.isArray(schema.indexes) ? schema.indexes : Object.values(schema.indexes || {}); + for (const index of indexesArray) { + const indexFields = Array.isArray(index.fields) + ? index.fields + : [index.fields]; + + // Simple heuristic: index is applicable if first field is in filter + if (indexFields.length > 0 && filterFields.has(indexFields[0])) { + const indexName = index.name || indexFields.join('_'); + indexes.push(indexName); + } + } + + return indexes; + } + + /** + * Analyze query for potential issues + * @private + */ + private analyzeWarnings(schema: ObjectConfig, query: UnifiedQuery): string[] { + const warnings: string[] = []; + + // Warning: No filters (full table scan) + if (!query.where || Object.keys(query.where).length === 0) { + warnings.push('No filters specified - this will scan all records'); + } + + // Warning: No limit on potentially large dataset + if (!query.limit) { + warnings.push('No limit specified - consider adding pagination'); + } + + // Warning: Selecting all fields + if (!query.fields || query.fields.length === 0) { + const fieldCount = Object.keys(schema.fields || {}).length; + if (fieldCount > 10) { + warnings.push(`Selecting all ${fieldCount} fields - consider selecting only needed fields`); + } + } + + // Warning: Complex filters without indexes + if (query.where && Object.keys(query.where).length > 5) { + const indexes = this.findApplicableIndexes(schema, query); + if (indexes.length === 0) { + warnings.push('Complex filters without matching indexes - consider adding indexes'); + } + } + + return warnings; + } + + /** + * Generate optimization suggestions + * @private + */ + private generateSuggestions( + schema: ObjectConfig, + query: UnifiedQuery, + indexes: string[] + ): string[] { + const suggestions: string[] = []; + + // Suggest adding limit + if (!query.limit) { + suggestions.push('Add a limit clause to restrict result set size'); + } + + // Suggest adding indexes + if (query.where && Object.keys(query.where).length > 0 && indexes.length === 0) { + const filterFields = Object.keys(query.where).filter(k => !k.startsWith('$')); + + if (filterFields.length > 0) { + suggestions.push(`Consider adding an index on: ${filterFields.join(', ')}`); + } + } + + // Suggest field selection + if (!query.fields || query.fields.length === 0) { + suggestions.push('Select only required fields to reduce data transfer'); + } + + // Suggest composite index for multiple filters + if (query.where && Object.keys(query.where).length > 1 && indexes.length < 2) { + const filterFields = Object.keys(query.where) + .filter(k => !k.startsWith('$')) + .slice(0, 3); // Top 3 fields + + if (filterFields.length > 1) { + suggestions.push(`Consider a composite index on: (${filterFields.join(', ')})`); + } + } + + return suggestions; + } + + /** + * Calculate query complexity score (0-100) + * @private + */ + private calculateComplexity(query: UnifiedQuery): number { + let complexity = 0; + + // Base complexity + complexity += 10; + + // Filters add complexity + if (query.where) { + const filterCount = Object.keys(query.where).length; + complexity += filterCount * 5; + + // Nested filters (OR/AND conditions) add more + const hasLogicalOps = query.where.$and || query.where.$or; + if (hasLogicalOps) { + complexity += 15; + } + } + + // Sorting adds complexity + if (query.orderBy && query.orderBy.length > 0) { + complexity += query.orderBy.length * 3; + } + + // Field selection reduces complexity slightly + if (query.fields && query.fields.length > 0 && query.fields.length < 10) { + complexity -= 5; + } + + // Pagination reduces complexity + if (query.limit) { + complexity -= 5; + } + + // Cap at 100 + return Math.min(Math.max(complexity, 0), 100); + } + + /** + * Estimate number of rows (very rough heuristic) + * @private + */ + private estimateRows(schema: ObjectConfig, query: UnifiedQuery): number { + // This is a placeholder - real implementation would need statistics + // from the database (row count, index selectivity, etc.) + + // Default to unknown + if (!query.where || Object.keys(query.where).length === 0) { + return -1; // Unknown, full scan + } + + // Very rough estimate based on filter count + const baseEstimate = 1000; + const filterCount = Object.keys(query.where).length; + const filterReduction = Math.pow(0.5, filterCount); + const estimated = Math.floor(baseEstimate * filterReduction); + + // Apply limit if present + if (query.limit) { + return Math.min(estimated, query.limit); + } + + return estimated; + } + + /** + * Record a query execution for statistics + * @private + */ + private recordExecution( + objectName: string, + executionTime: number, + query: UnifiedQuery + ): void { + // Update totals + this.stats.totalQueries++; + this.executionTimes.push(executionTime); + + // Update average + this.stats.avgExecutionTime = + this.executionTimes.reduce((sum, t) => sum + t, 0) / this.executionTimes.length; + + // Update min/max + this.stats.slowestQuery = Math.max(this.stats.slowestQuery, executionTime); + this.stats.fastestQuery = Math.min(this.stats.fastestQuery, executionTime); + + // Update per-object stats + if (!this.stats.byObject[objectName]) { + this.stats.byObject[objectName] = { count: 0, avgTime: 0 }; + } + const objStats = this.stats.byObject[objectName]; + objStats.count++; + objStats.avgTime = ((objStats.avgTime * (objStats.count - 1)) + executionTime) / objStats.count; + + // Track slow queries + if (this.stats.slowQueries.length < this.MAX_SLOW_QUERIES) { + this.stats.slowQueries.push({ + objectName, + executionTime, + query, + timestamp: new Date() + }); + this.stats.slowQueries.sort((a, b) => b.executionTime - a.executionTime); + } else if (executionTime > this.stats.slowQueries[this.MAX_SLOW_QUERIES - 1].executionTime) { + this.stats.slowQueries[this.MAX_SLOW_QUERIES - 1] = { + objectName, + executionTime, + query, + timestamp: new Date() + }; + this.stats.slowQueries.sort((a, b) => b.executionTime - a.executionTime); + } + } +} diff --git a/packages/foundation/plugin-query/src/query-builder.ts b/packages/foundation/plugin-query/src/query-builder.ts new file mode 100644 index 00000000..47c6c966 --- /dev/null +++ b/packages/foundation/plugin-query/src/query-builder.ts @@ -0,0 +1,64 @@ +/** + * ObjectQL + * Copyright (c) 2026-present ObjectStack Inc. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +import type { UnifiedQuery, QueryAST } from '@objectql/types'; +// import { QueryAST } from './types'; (Removed) + +// Local QueryAST type extension to include all properties we need +// interface QueryAST extends Data.QueryAST { +// top?: number; +// expand?: Record; +// aggregations?: any[]; +// having?: any; +// } + +import { FilterTranslator } from './filter-translator'; + +/** + * Query Builder + * + * Builds ObjectStack QueryAST from ObjectQL UnifiedQuery. + * Since UnifiedQuery now uses the standard protocol format directly, + * this is now a simple pass-through with object name injection. + */ +export class QueryBuilder { + private filterTranslator: FilterTranslator; + + constructor() { + this.filterTranslator = new FilterTranslator(); + } + + /** + * Build a QueryAST from a UnifiedQuery + * + * @param objectName - Target object name + * @param query - ObjectQL UnifiedQuery (now in standard QueryAST format) + * @returns ObjectStack QueryAST + */ + build(objectName: string, query: UnifiedQuery): QueryAST { + // UnifiedQuery now uses the same format as QueryAST + // Just add the object name and pass through + const ast: QueryAST = { + object: objectName + }; + + // Map UnifiedQuery properties to QueryAST + if (query.fields) ast.fields = query.fields; + if (query.where) ast.where = this.filterTranslator.translate(query.where); + if (query.orderBy) ast.orderBy = query.orderBy; + if (query.offset !== undefined) ast.offset = query.offset; + if (query.limit !== undefined) ast.top = query.limit; // UnifiedQuery uses 'limit', QueryAST uses 'top' + if (query.expand) ast.expand = query.expand; + if (query.groupBy) ast.groupBy = query.groupBy; + if (query.aggregations) ast.aggregations = query.aggregations; + if (query.having) ast.having = query.having; + if (query.distinct) ast.distinct = query.distinct; + + return ast; + } +} diff --git a/packages/foundation/plugin-query/src/query-service.ts b/packages/foundation/plugin-query/src/query-service.ts new file mode 100644 index 00000000..1b305b19 --- /dev/null +++ b/packages/foundation/plugin-query/src/query-service.ts @@ -0,0 +1,397 @@ +/** + * ObjectQL Query Service + * Copyright (c) 2026-present ObjectStack Inc. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +import type { + Driver, + ObjectConfig, + UnifiedQuery, + Filter, + MetadataRegistry +} from '@objectql/types'; +import { QueryAST, ObjectQLError } from '@objectql/types'; +import { QueryBuilder } from './query-builder'; +import { QueryCompiler } from '@objectql/plugin-optimizations'; + +/** + * Options for query execution + */ +export interface QueryOptions { + /** + * Transaction handle for transactional queries + */ + transaction?: any; + + /** + * Skip validation (for system operations) + */ + skipValidation?: boolean; + + /** + * Include profiling information + */ + profile?: boolean; + + /** + * Custom driver options + */ + driverOptions?: Record; +} + +/** + * Result of a query execution with optional profiling data + */ +export interface QueryResult { + /** + * The query results + */ + value: T; + + /** + * Total count (for paginated queries) + */ + count?: number; + + /** + * Profiling information (if profile option was enabled) + */ + profile?: QueryProfile; +} + +/** + * Profiling information for a query + */ +export interface QueryProfile { + /** + * Execution time in milliseconds + */ + executionTime: number; + + /** + * Number of rows scanned + */ + rowsScanned?: number; + + /** + * Whether an index was used + */ + indexUsed?: boolean; + + /** + * The generated QueryAST + */ + ast?: QueryAST; +} + +/** + * Query Service + * + * Handles all query execution logic, separating query concerns from + * the repository pattern. This service is responsible for: + * - Building QueryAST from UnifiedQuery + * - Executing queries via drivers + * - Optional query profiling and analysis + * + * The QueryService is registered as a service in the ObjectQLPlugin + * and can be used by Repository for all read operations. + */ +export class QueryService { + private queryBuilder: QueryBuilder; + private queryCompiler: QueryCompiler; + + constructor( + private datasources: Record, + private metadata: MetadataRegistry + ) { + this.queryBuilder = new QueryBuilder(); + this.queryCompiler = new QueryCompiler(1000); + } + + /** + * Get the driver for a specific object + * @private + */ + private getDriver(objectName: string): Driver { + const obj = this.getSchema(objectName); + const datasourceName = obj.datasource || 'default'; + const driver = this.datasources[datasourceName]; + + if (!driver) { + throw new ObjectQLError({ code: 'NOT_FOUND', message: `Datasource '${datasourceName}' not found for object '${objectName}'` }); + } + + return driver; + } + + /** + * Get the schema for an object + * @private + */ + private getSchema(objectName: string): ObjectConfig { + const obj = this.metadata.get('object', objectName); + if (!obj) { + throw new ObjectQLError({ code: 'NOT_FOUND', message: `Object '${objectName}' not found in metadata` }); + } + return obj; + } + + /** + * Build QueryAST from UnifiedQuery + * @private + */ + private buildQueryAST(objectName: string, query: UnifiedQuery): QueryAST { + const ast = this.queryBuilder.build(objectName, query); + const compiled = this.queryCompiler.compile(objectName, ast); + return compiled.ast; + } + + /** + * Execute a find query + * + * @param objectName - The object to query + * @param query - The unified query + * @param options - Query execution options + * @returns Array of matching records + */ + async find( + objectName: string, + query: UnifiedQuery = {}, + options: QueryOptions = {} + ): Promise> { + const driver = this.getDriver(objectName); + const startTime = options.profile ? Date.now() : 0; + + // Build QueryAST + const ast = this.buildQueryAST(objectName, query); + + // Execute query via driver + const driverOptions = { + transaction: options.transaction, + ...options.driverOptions + }; + + let results: any[]; + let count: number | undefined; + + if (driver.find) { + // Legacy driver interface + const result: any = await driver.find(objectName, query, driverOptions); + results = Array.isArray(result) ? result : (result?.value || []); + count = (typeof result === 'object' && !Array.isArray(result) && result?.count !== undefined) ? result.count : undefined; + } else if (driver.executeQuery) { + // New DriverInterface + const result = await driver.executeQuery(ast, driverOptions); + results = result.value || []; + count = result.count; + } else { + throw new ObjectQLError({ code: 'DRIVER_UNSUPPORTED_OPERATION', message: `Driver does not support query execution` }); + } + + const executionTime = options.profile ? Date.now() - startTime : 0; + + return { + value: results, + count, + profile: options.profile ? { + executionTime, + ast, + rowsScanned: results.length, + } : undefined + }; + } + + /** + * Execute a findOne query by ID + * + * @param objectName - The object to query + * @param id - The record ID + * @param options - Query execution options + * @returns The matching record or undefined + */ + async findOne( + objectName: string, + id: string | number, + options: QueryOptions = {} + ): Promise> { + const driver = this.getDriver(objectName); + const startTime = options.profile ? Date.now() : 0; + + const driverOptions = { + transaction: options.transaction, + ...options.driverOptions + }; + + let result: any; + + if (driver.findOne) { + // Legacy driver interface + result = await driver.findOne(objectName, id, driverOptions); + } else if (driver.get) { + // Alternative method name + result = await driver.get(objectName, String(id), driverOptions); + } else if (driver.executeQuery) { + // Fallback to query with ID filter + const query: UnifiedQuery = { + where: { _id: id } + }; + const ast = this.buildQueryAST(objectName, query); + const queryResult = await driver.executeQuery(ast, driverOptions); + result = queryResult.value?.[0]; + } else { + throw new ObjectQLError({ code: 'DRIVER_UNSUPPORTED_OPERATION', message: `Driver does not support findOne operation` }); + } + + const executionTime = options.profile ? Date.now() - startTime : 0; + + return { + value: result, + profile: options.profile ? { + executionTime, + rowsScanned: result ? 1 : 0, + } : undefined + }; + } + + /** + * Execute a count query + * + * @param objectName - The object to query + * @param where - Optional filter condition + * @param options - Query execution options + * @returns Count of matching records + */ + async count( + objectName: string, + where?: Filter, + options: QueryOptions = {} + ): Promise> { + const driver = this.getDriver(objectName); + const startTime = options.profile ? Date.now() : 0; + + const query: UnifiedQuery = where ? { where } : {}; + const ast = this.buildQueryAST(objectName, query); + + const driverOptions = { + transaction: options.transaction, + ...options.driverOptions + }; + + let count: number; + + if (driver.count) { + // Legacy driver interface + count = await driver.count(objectName, where || {}, driverOptions); + } else if (driver.executeQuery) { + // Use executeQuery and count results + // Note: This is inefficient for large datasets + // Ideally, driver should support count-specific optimization + const result = await driver.executeQuery(ast, driverOptions); + count = result.count ?? result.value?.length ?? 0; + } else { + throw new ObjectQLError({ code: 'DRIVER_UNSUPPORTED_OPERATION', message: `Driver does not support count operation` }); + } + + const executionTime = options.profile ? Date.now() - startTime : 0; + + return { + value: count, + profile: options.profile ? { + executionTime, + ast, + } : undefined + }; + } + + /** + * Execute an aggregate query + * + * @param objectName - The object to query + * @param query - The aggregation query using UnifiedQuery format + * @param options - Query execution options + * @returns Aggregation results + */ + async aggregate( + objectName: string, + query: UnifiedQuery, + options: QueryOptions = {} + ): Promise> { + const driver = this.getDriver(objectName); + const startTime = options.profile ? Date.now() : 0; + + const driverOptions = { + transaction: options.transaction, + ...options.driverOptions + }; + + let results: any[]; + + if (driver.aggregate) { + // Driver supports aggregation + results = await driver.aggregate(objectName, query, driverOptions); + } else { + // Driver doesn't support aggregation + throw new ObjectQLError({ code: 'DRIVER_UNSUPPORTED_OPERATION', message: `Driver does not support aggregate operations. Consider using a driver that supports aggregation.` }); + } + + const executionTime = options.profile ? Date.now() - startTime : 0; + + return { + value: results, + profile: options.profile ? { + executionTime, + rowsScanned: results.length, + } : undefined + }; + } + + /** + * Execute a direct SQL/query passthrough + * + * This bypasses ObjectQL's query builder and executes raw queries. + * Use with caution as it bypasses security and validation. + * + * @param objectName - The object (determines which datasource to use) + * @param queryString - Raw query string (SQL, MongoDB query, etc.) + * @param params - Query parameters (for parameterized queries) + * @param options - Query execution options + * @returns Query results + */ + async directQuery( + objectName: string, + queryString: string, + params?: any[], + options: QueryOptions = {} + ): Promise> { + const driver = this.getDriver(objectName); + const startTime = options.profile ? Date.now() : 0; + + const _driverOptions = { + transaction: options.transaction, + ...options.driverOptions + }; + + let results: any; + + if (driver.directQuery) { + results = await driver.directQuery(queryString, params); + } else if (driver.query) { + // Alternative method name + results = await driver.query(queryString, params); + } else { + throw new ObjectQLError({ code: 'DRIVER_UNSUPPORTED_OPERATION', message: `Driver does not support direct query execution` }); + } + + const executionTime = options.profile ? Date.now() - startTime : 0; + + return { + value: results, + profile: options.profile ? { + executionTime, + } : undefined + }; + } +} diff --git a/packages/foundation/plugin-query/tsconfig.json b/packages/foundation/plugin-query/tsconfig.json new file mode 100644 index 00000000..50a8f6e0 --- /dev/null +++ b/packages/foundation/plugin-query/tsconfig.json @@ -0,0 +1,13 @@ +{ + "extends": "../../../tsconfig.base.json", + "compilerOptions": { + "outDir": "dist", + "rootDir": "src" + }, + "include": ["src/**/*"], + "exclude": ["node_modules", "dist"], + "references": [ + { "path": "../types" }, + { "path": "../plugin-optimizations" } + ] +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 731b4885..997908c1 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -549,6 +549,12 @@ importers: '@objectql/plugin-formula': specifier: workspace:* version: link:../plugin-formula + '@objectql/plugin-optimizations': + specifier: workspace:* + version: link:../plugin-optimizations + '@objectql/plugin-query': + specifier: workspace:* + version: link:../plugin-query '@objectql/plugin-validator': specifier: workspace:* version: link:../plugin-validator @@ -654,6 +660,41 @@ importers: specifier: ^5.3.0 version: 5.9.3 + packages/foundation/plugin-optimizations: + dependencies: + '@objectql/types': + specifier: workspace:* + version: link:../types + '@objectstack/core': + specifier: ^2.0.6 + version: 2.0.6 + '@objectstack/spec': + specifier: ^2.0.6 + version: 2.0.6 + devDependencies: + typescript: + specifier: ^5.3.0 + version: 5.9.3 + + packages/foundation/plugin-query: + dependencies: + '@objectql/plugin-optimizations': + specifier: workspace:* + version: link:../plugin-optimizations + '@objectql/types': + specifier: workspace:* + version: link:../types + '@objectstack/core': + specifier: ^2.0.6 + version: 2.0.6 + '@objectstack/spec': + specifier: ^2.0.6 + version: 2.0.6 + devDependencies: + typescript: + specifier: ^5.3.0 + version: 5.9.3 + packages/foundation/plugin-security: dependencies: '@objectql/types': diff --git a/vitest.config.ts b/vitest.config.ts index 94182e87..d6b7caff 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -26,6 +26,8 @@ export default defineConfig({ '@objectql/plugin-formula': path.resolve(__dirname, './packages/foundation/plugin-formula/src'), '@objectql/plugin-security': path.resolve(__dirname, './packages/foundation/plugin-security/src'), '@objectql/plugin-sync': path.resolve(__dirname, './packages/foundation/plugin-sync/src'), + '@objectql/plugin-query': path.resolve(__dirname, './packages/foundation/plugin-query/src'), + '@objectql/plugin-optimizations': path.resolve(__dirname, './packages/foundation/plugin-optimizations/src'), '@objectql/edge-adapter': path.resolve(__dirname, './packages/foundation/edge-adapter/src'), '@objectql/protocol-graphql': path.resolve(__dirname, './packages/protocols/graphql/src'), '@objectql/protocol-odata-v4': path.resolve(__dirname, './packages/protocols/odata-v4/src'),