From 789b0cd1bf6b76d3469bd03265958f1b9bf15515 Mon Sep 17 00:00:00 2001 From: Bob Weinand Date: Tue, 15 Mar 2022 00:25:02 +0100 Subject: [PATCH 1/5] Add #include_next support --- lib/Context.php | 67 +++++++++++++++++++++++++++++++++++++++++++- lib/PreProcessor.php | 51 +++++++++++++-------------------- 2 files changed, 86 insertions(+), 32 deletions(-) diff --git a/lib/Context.php b/lib/Context.php index 1ed043b..b72fe0b 100755 --- a/lib/Context.php +++ b/lib/Context.php @@ -44,6 +44,38 @@ private function locateGCCHeaderPaths() { } } + public function findHeaderFile(string $header, string $contextDir, string $contextFile, bool $next): ?string { + if ($header[0] === '/' || ($header[1] === ':' && $header[2] === '\\')) { + if (file_exists($header)) { + return $header; + } + } else { + if ($contextDir && !$next) { + $dir = $contextDir; + while (!empty($dir) && $dir !== '/') { + $file = "$dir/$header"; + if (file_exists($file)) { + return $file; + } + $dir = dirname($dir); + } + } + foreach ($this->headerSearchPaths as $path) { + if ($next) { + if ($contextDir === $path) { + $next = false; + } + break; + } + $test = $path . '/' . $header; + if (file_exists($test)) { + return $test; + } + } + } + return null; + } + public function getDefines(): array { return $this->definitions; } @@ -142,7 +174,7 @@ public function evaluate(?Token $expr): Token { return new Token(Token::NUMBER, '0', 'computed'); } return $expr; - } + } list ($result, $expr) = $this->evaluateInternal($expr); if ($expr !== null) { throw new \LogicException('Syntax error: unknown trailing expr: ' . $expr->value); @@ -187,6 +219,38 @@ public function evaluateInternal(?Token $expr, bool $single = false): array { } else { throw new \LogicException("Syntax Error for #defined expression, expecting ( or IDENTIFIER, found " . $expr->value); } + } elseif ($expr->type === Token::IDENTIFIER && $expr->value === '__has_include') { + $expr = Token::skipWhitespace($expr->next); + if ($expr === null) { + throw new \LogicException("Syntax Error for __has_include() expression: not enough tokens"); + } + if ($expr->type === Token::PUNCTUATOR && $expr->value === '(') { + $expr = Token::skipWhitespace($expr->next); + if ($expr->type === Token::LITERAL) { + $file = $expr->value; + } elseif ($expr->type === Token::PUNCTUATOR && $expr->value === '<') { + // handle <> include: + $file = ''; + while (!empty($expr->next)) { + $expr = $expr->next; + if ($expr->type === Token::PUNCTUATOR && $expr->value === '>') { + break; + } + $file .= $expr->value; + } + } else { + throw new \LogicException("Syntax Error for __has_include() expression, expecting < or LITERAL, found " . $expr->value); + } + $expr = Token::skipWhitespace($expr->next); + if ($expr === null || $expr->type !== Token::PUNCTUATOR && $expr->value !== ')') { + throw new \LogicException("Syntax Error for __has_include() expression: ) not found"); + } + $contextDir = dirname($expr->file); + $result = new Token(Token::NUMBER, $this->findHeaderFile($file, $contextDir, $expr->file, false) !== null ? '1' : '0', 'computed'); + $expr = Token::skipWhitespace($expr->next); + } else { + throw new \LogicException("Syntax Error for #__has_include expression, expecting (, found " . $expr->value); + } } elseif ($expr->type === Token::IDENTIFIER) { $next = Token::skipWhitespace($expr->next); if ($next !== null && $next->value === '(') { @@ -221,6 +285,7 @@ public function evaluateInternal(?Token $expr, bool $single = false): array { $result = new Token(Token::NUMBER, $expr->value, 'computed'); $expr = Token::skipWhitespace($expr->next); } else { + var_dump($expr); throw new \LogicException('Unknown operator ' . $expr->value); } if ($negate) { diff --git a/lib/PreProcessor.php b/lib/PreProcessor.php index f08761f..39e1bf5 100755 --- a/lib/PreProcessor.php +++ b/lib/PreProcessor.php @@ -50,6 +50,10 @@ public function process(string $header): array { $tokens = $this->resolveInclude($line, $directive->file); $lines = array_merge($tokens, $lines); break; + case 'include_next': + $tokens = $this->resolveInclude($line, $directive->file, true); + $lines = array_merge($tokens, $lines); + break; case 'define': if (empty($line)) { throw new \LogicException("#define must have a name"); @@ -113,6 +117,7 @@ public function process(string $header): array { $this->debug($directive); throw new \LogicException('We reached an error preprocessor token:'); default: + var_dump($line); var_dump($directive->value); throw new \LogicException("Unknown directive found {$directive->value}"); } @@ -187,15 +192,15 @@ private function skipIf(array $lines, bool $skipAll = false): array { return []; } - private function findAndParse(string $header, string $contextDir, string $contextFile): array { + private function findAndParse(string $header, string $contextDir, string $contextFile, bool $next = false): array { $contextDir = rtrim($contextDir, '/'); - $file = $this->findHeaderFile($header, $contextDir, $contextFile); + $file = $this->findHeaderFile($header, $contextDir, $contextFile, $next); $code = file_get_contents($file); $lines = $this->parser->parse($file, $code); return $lines; } - private function resolveInclude(?Token $arg, string $contextFile): array { + private function resolveInclude(?Token $arg, string $contextFile, bool $next = false): array { $contextDir = dirname($contextFile); if (empty($arg)) { throw new \LogicException("Empty include declaration"); @@ -204,9 +209,9 @@ private function resolveInclude(?Token $arg, string $contextFile): array { if ($type->type === Token::LITERAL) { $file = $type->value; if (!empty($args)) { - throw new \LogicException("extra tokens in #include directive"); + throw new \LogicException("extra tokens in #include" . ($next ? "_next" : "") . " directive"); } - return $this->findAndParse($file, $contextDir, $contextFile); + return $this->findAndParse($file, $contextDir, $contextFile, $next); } elseif ($type->type === Token::PUNCTUATOR && $type->value === '<' && !empty($arg->next)) { // handle <> include: $file = ''; @@ -218,37 +223,18 @@ private function resolveInclude(?Token $arg, string $contextFile): array { $file .= $arg->value; } if (!empty($args)) { - throw new \LogicException("extra tokens in #include directive"); + throw new \LogicException("extra tokens in #include" . ($next ? "_next" : "") . " directive"); } // always a system import - return $this->findAndParse($file, $contextDir, $contextFile); + return $this->findAndParse($file, $contextDir, $contextFile, $next); } var_dump($type, $arg); throw new \LogicException("Illegal include directive"); } - private function findHeaderFile(string $header, string $contextDir, string $contextFile): string { - if ($header[0] === '/' || ($header[1] === ':' && $header[2] === '\\')) { - if (file_exists($header)) { - return $header; - } - } else { - if ($contextDir) { - $dir = $contextDir; - while (!empty($dir) && $dir !== '/') { - $file = "$dir/$header"; - if (file_exists($file)) { - return $file; - } - $dir = dirname($dir); - } - } - foreach ($this->context->headerSearchPaths as $path) { - $test = $path . '/' . $header; - if (file_exists($test)) { - return $test; - } - } + private function findHeaderFile(string $header, string $contextDir, string $contextFile, bool $next): string { + if ($headerFile = $this->context->findHeaderFile($header, $contextDir, $contextFile, $next)) { + return $headerFile; } var_dump($this->context->headerSearchPaths); throw new \LogicException("Could not find header file: $header given context $contextDir (called from $contextFile)"); @@ -256,6 +242,9 @@ private function findHeaderFile(string $header, string $contextDir, string $cont private function debug(?Token $token): void { echo "T: "; + if ($token) { + echo "@{$token->file} :"; + } while ($token !== null) { echo $token->value . ' '; $token = $token->next; @@ -263,7 +252,7 @@ private function debug(?Token $token): void { echo "\n"; } - + private function prepareAndDoCall(string $identifier, array $rawargs): ?Token { $args = []; $first = null; @@ -313,7 +302,7 @@ private function expandMacros(?Token $expr, int $recurseLevel = 0): ?Token { $this->callStack = new CallStack($expr->value, $this->callStack); $result = $this->callStack->currentArg; goto next; - } + } // It's not a call, so treat it literally $result = $result->next = new Token($expr->type, $expr->value, $expr->file); goto next; From 447ebae1a81b75815088748fdb6b09f05438b99f Mon Sep 17 00:00:00 2001 From: Bob Weinand Date: Tue, 15 Mar 2022 00:26:06 +0100 Subject: [PATCH 2/5] Fix #elif chaining in subsequent lines --- lib/PreProcessor.php | 2 +- test/cases/c/elifchain.phpt | 16 ++++++++++++++ test/generated/c/elifchainTest.c | 11 ++++++++++ test/generated/c/elifchainTest.php | 34 ++++++++++++++++++++++++++++++ 4 files changed, 62 insertions(+), 1 deletion(-) create mode 100755 test/cases/c/elifchain.phpt create mode 100644 test/generated/c/elifchainTest.c create mode 100644 test/generated/c/elifchainTest.php diff --git a/lib/PreProcessor.php b/lib/PreProcessor.php index 39e1bf5..5a7121b 100755 --- a/lib/PreProcessor.php +++ b/lib/PreProcessor.php @@ -104,7 +104,7 @@ public function process(string $header): array { break; case 'else': case 'elif': - $lines = $this->skipIf($lines); + $lines = $this->skipIf($lines, true); break; case 'endif': // ignore diff --git a/test/cases/c/elifchain.phpt b/test/cases/c/elifchain.phpt new file mode 100755 index 0000000..a0ea5c2 --- /dev/null +++ b/test/cases/c/elifchain.phpt @@ -0,0 +1,16 @@ +--TEST-- +Test for #elif chains +--FILE-- + +#if 0 +#elif 1 +#elif 0 +#elif 0 +#else +#error "ERROR" +#endif + +int bar; + +--EXPECT-- +int bar; diff --git a/test/generated/c/elifchainTest.c b/test/generated/c/elifchainTest.c new file mode 100644 index 0000000..f89be25 --- /dev/null +++ b/test/generated/c/elifchainTest.c @@ -0,0 +1,11 @@ + +#if 0 +#elif 1 +#elif 0 +#elif 0 +#else +#error "ERROR" +#endif + +int bar; + diff --git a/test/generated/c/elifchainTest.php b/test/generated/c/elifchainTest.php new file mode 100644 index 0000000..91afa89 --- /dev/null +++ b/test/generated/c/elifchainTest.php @@ -0,0 +1,34 @@ +parser = new CParser; + $this->parser->addSearchPath(__DIR__); + $this->parser->addSearchPath(__DIR__ . '/../../include'); + $this->printer = new C; + } + + /** + * @textdox Test for #elif chains + */ + public function testCode() { + $translationUnit = $this->parser->parse(__DIR__ . '/elifchainTest.c'); + $actual = $this->printer->print($translationUnit); + $this->assertEquals(self::EXPECTED, trim($actual)); + } +} \ No newline at end of file From 2407dd92ff7ed2d20b27fa9263fa8406ecc9ebbf Mon Sep 17 00:00:00 2001 From: Bob Weinand Date: Tue, 15 Mar 2022 00:26:55 +0100 Subject: [PATCH 3/5] Add variadic macro / __VA_ARGS__ support --- lib/Context.php | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/lib/Context.php b/lib/Context.php index b72fe0b..2bbaeba 100755 --- a/lib/Context.php +++ b/lib/Context.php @@ -454,7 +454,28 @@ public function doCall(string $toCall, ?Token ...$args): Token { $argIdx = 0; if ($token->value === '(') { $token = Token::skipWhitespace($token->next); + $isVariadic = false; while ($token !== null && $token->value !== ')') { + if ($isVariadic) { + throw new \LogicException('Unexpected token found, expecting ) after ... found ' . $token->value); + } + if ($token->type === Token::PUNCTUATOR && $token->value === "...") { + $isVariadic = true; + if (isset($args[$argIdx])) { + $argMap["__VA_ARGS__"] = $variadic = $args[$argIdx++]; + while (isset($args[$argIdx])) { + while ($variadic->next) { + $variadic = $variadic->next; + } + $variadic->next = $args[$argIdx++]; + } + } else { + $argMap["__VA_ARGS__"] = new Token(Token::OTHER, '', 'computed'); + } + $token = Token::skipWhitespace($token->next); + continue; + } + if ($token->type !== Token::IDENTIFIER) { throw new \LogicException('Unexpected argument found, expecting IDENTIFIER found ' . $token->value); } elseif (!array_key_exists($argIdx, $args)) { @@ -478,6 +499,16 @@ public function doCall(string $toCall, ?Token ...$args): Token { // Copy token stream $first = $newToken = new Token(0, '', 'internal'); while ($token !== null) { + // handle , ##__VA_ARGS__ + if ($token->type === Token::PUNCTUATOR && $token->value === ',') { + $nextToken = Token::skipWhitespace($token); + if ($nextToken->type === Token::PUNCTUATOR && $token->value === '##') { + if (\count($argMap) > $argIdx) { + $newToken = $newToken->next = new Token($token->type, $token->value, $token->file); + } + $token = Token::skipWhitespace($nextToken); + } + } if ($token->type === Token::IDENTIFIER && array_key_exists($token->value, $argMap)) { $arg = $argMap[$token->value]; $toAdd = $toAddNext = new Token(Token::OTHER, '', 'computed'); From 191d5f6285feaa9f3a73c0247ceafbfdb1c73010 Mon Sep 17 00:00:00 2001 From: Bob Weinand Date: Tue, 15 Mar 2022 00:27:58 +0100 Subject: [PATCH 4/5] Handle unsigned numbers E.g. 0xffU --- lib/Context.php | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/Context.php b/lib/Context.php index 2bbaeba..30b25f4 100755 --- a/lib/Context.php +++ b/lib/Context.php @@ -427,6 +427,8 @@ private function normalize(Token $expr): int { throw new \LogicException("Base mismatch for {$str}, found $chr for $idx"); } $result = ($result * $base) + $chr; + } elseif ($str[$idx] === 'U') { + // unsigned number, let's not touch it } elseif ($str[$idx] === 'L') { // indicates number is a long, return as is if ($idx + 1 !== $length) { From 134668d72bb25073ca869237968dc9a929fd70ed Mon Sep 17 00:00:00 2001 From: Bob Weinand Date: Tue, 15 Mar 2022 00:55:06 +0100 Subject: [PATCH 5/5] Implement pragma once Skip all other pragmas --- lib/PreProcessor.php | 28 +++++++++++++++++++---- test/cases/c/pragma_once.phpt | 11 +++++++++ test/generated/c/pragma_onceTest.c | 6 +++++ test/generated/c/pragma_onceTest.php | 34 ++++++++++++++++++++++++++++ test/include/pragma_once.h | 8 +++++++ 5 files changed, 82 insertions(+), 5 deletions(-) create mode 100755 test/cases/c/pragma_once.phpt create mode 100644 test/generated/c/pragma_onceTest.c create mode 100644 test/generated/c/pragma_onceTest.php create mode 100644 test/include/pragma_once.h diff --git a/lib/PreProcessor.php b/lib/PreProcessor.php index 5a7121b..6ad936f 100755 --- a/lib/PreProcessor.php +++ b/lib/PreProcessor.php @@ -109,11 +109,24 @@ public function process(string $header): array { case 'endif': // ignore break; + case 'pragma': + if (empty($line)) { + throw new \LogicException("At least one declaration is required for pragma"); + } + $pragmaMode = $line; + if ($pragmaMode->value === "once") { + if (Token::skipWhitespace($pragmaMode->next)) { + throw new \LogicException("pragma once has no further arguments"); + } + $this->headers[$pragmaMode->file] = true; + } + break; case 'warning': // ignore break; case 'error': - var_dump($this->context); + //var_dump($this->context); + //var_dump(array_keys($this->context->getDefines())); $this->debug($directive); throw new \LogicException('We reached an error preprocessor token:'); default: @@ -178,6 +191,7 @@ private function skipIf(array $lines, bool $skipAll = false): array { return $lines; case 'define': case 'include': + case 'pragma': // there are no special pragmas supposed to be in #if's case 'undef': case 'error': case 'warning': @@ -195,6 +209,10 @@ private function skipIf(array $lines, bool $skipAll = false): array { private function findAndParse(string $header, string $contextDir, string $contextFile, bool $next = false): array { $contextDir = rtrim($contextDir, '/'); $file = $this->findHeaderFile($header, $contextDir, $contextFile, $next); + if (($this->headers[$file] ?? false) === true) { // has pragma once + return []; + } + $this->headers[$file] = false; $code = file_get_contents($file); $lines = $this->parser->parse($file, $code); return $lines; @@ -235,7 +253,7 @@ private function resolveInclude(?Token $arg, string $contextFile, bool $next = f private function findHeaderFile(string $header, string $contextDir, string $contextFile, bool $next): string { if ($headerFile = $this->context->findHeaderFile($header, $contextDir, $contextFile, $next)) { return $headerFile; - } + } var_dump($this->context->headerSearchPaths); throw new \LogicException("Could not find header file: $header given context $contextDir (called from $contextFile)"); } @@ -252,7 +270,7 @@ private function debug(?Token $token): void { echo "\n"; } - + private function prepareAndDoCall(string $identifier, array $rawargs): ?Token { $args = []; $first = null; @@ -302,7 +320,7 @@ private function expandMacros(?Token $expr, int $recurseLevel = 0): ?Token { $this->callStack = new CallStack($expr->value, $this->callStack); $result = $this->callStack->currentArg; goto next; - } + } // It's not a call, so treat it literally $result = $result->next = new Token($expr->type, $expr->value, $expr->file); goto next; @@ -390,4 +408,4 @@ public function nextArg(): void { $this->args[] = $this->currentArg->next; $this->currentArg = new Token(0, '', 'internal'); } -} +} \ No newline at end of file diff --git a/test/cases/c/pragma_once.phpt b/test/cases/c/pragma_once.phpt new file mode 100755 index 0000000..43c7187 --- /dev/null +++ b/test/cases/c/pragma_once.phpt @@ -0,0 +1,11 @@ +--TEST-- +Test for #elif chains +--FILE-- + +#include "pragma_once.h" +#include "pragma_once.h" + +int TEST; + +--EXPECT-- +int bar; diff --git a/test/generated/c/pragma_onceTest.c b/test/generated/c/pragma_onceTest.c new file mode 100644 index 0000000..f4b74e2 --- /dev/null +++ b/test/generated/c/pragma_onceTest.c @@ -0,0 +1,6 @@ + +#include "pragma_once.h" +#include "pragma_once.h" + +int TEST; + diff --git a/test/generated/c/pragma_onceTest.php b/test/generated/c/pragma_onceTest.php new file mode 100644 index 0000000..3da2e55 --- /dev/null +++ b/test/generated/c/pragma_onceTest.php @@ -0,0 +1,34 @@ +parser = new CParser; + $this->parser->addSearchPath(__DIR__); + $this->parser->addSearchPath(__DIR__ . '/../../include'); + $this->printer = new C; + } + + /** + * @textdox Test for #elif chains + */ + public function testCode() { + $translationUnit = $this->parser->parse(__DIR__ . '/pragma_onceTest.c'); + $actual = $this->printer->print($translationUnit); + $this->assertEquals(self::EXPECTED, trim($actual)); + } +} \ No newline at end of file diff --git a/test/include/pragma_once.h b/test/include/pragma_once.h new file mode 100644 index 0000000..8171ef2 --- /dev/null +++ b/test/include/pragma_once.h @@ -0,0 +1,8 @@ + +#ifdef TEST +#error "VERY BAD" +#endif + +#pragma once + +#define TEST "bar"