Skip to content

Asynchronous file handling #72

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
thekid merged 6 commits into from
Mar 31, 2021
Merged

Asynchronous file handling #72

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
0f45c49
Adjust test to be able to handle async file handling
thekid Mar 31, 2021
63fc3cb
Yield control after transferring each chunk of the given file
thekid Mar 31, 2021
47c0d95
Simplify code by passing range to copy()
thekid Mar 31, 2021
a3d85e3
Ensure file is closed
thekid Mar 31, 2021
835ef55
Ensure file and output are closed
thekid Mar 31, 2021
01d4717
Use the same chunk size for all transfers
thekid Mar 31, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
56 changes: 34 additions & 22 deletions src/main/php/web/handler/FilesFrom.class.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,8 @@
use web\io\Ranges;

class FilesFrom implements Handler {
const BOUNDARY = '594fa07300f865fe';
const BOUNDARY = '594fa07300f865fe';
const CHUNKSIZE = 8192;

private $path;

Expand Down Expand Up @@ -45,20 +46,25 @@ public function handle($request, $response) {
$file= $target->asFile();
}

$this->serve($request, $response, $file);
return $this->serve($request, $response, $file);
}

/**
* Copies a given amount of bytes from the specified file to the output
*
* @param web.io.Output $output
* @param io.File $file
* @param int $length
* @param web.io.Range $range
* @return iterable
*/
private function copy($output, $file, $length) {
while ($length && $chunk= $file->read(min(8192, $length))) {
private function copy($output, $file, $range) {
$file->seek($range->start());

$length= $range->length();
while ($length && $chunk= $file->read(min(self::CHUNKSIZE, $length))) {
$output->write($chunk);
$length-= strlen($chunk);
yield;
}
}

Expand Down Expand Up @@ -94,7 +100,19 @@ public function serve($request, $response, $target) {
$mimeType= MimeType::getByFileName($file->filename);
if (null === ($ranges= Ranges::in($request->header('Range'), $file->size()))) {
$response->answer(200, 'OK');
$response->transfer($file->in(), $mimeType, $file->size());
$response->header('Content-Type', $mimeType);

$out= $response->stream($file->size());
$file->open(File::READ);
try {
do {
$out->write($file->read(self::CHUNKSIZE));
yield;
} while (!$file->eof());
} finally {
$file->close();
$out->close();
}
return;
}

Expand All @@ -106,47 +124,41 @@ public function serve($request, $response, $target) {
}

$file->open(File::READ);
$output= $response->output();
$response->answer(206, 'Partial Content');

try {
if ($range= $ranges->single()) {
$response->header('Content-Type', $mimeType);
$response->header('Content-Range', $ranges->format($range));
$response->header('Content-Length', $range->length());

$file->seek($range->start());
$response->flush();
$this->copy($output, $file, $range->length());
$out= $response->stream($range->length());
yield from $this->copy($out, $file, $range);
} else {
$headers= [];
$trailer= "\r\n--".self::BOUNDARY."--\r\n";

$length= strlen($trailer);

foreach ($ranges->sets() as $i => $range) {
$header= sprintf(
$headers[$i]= $header= sprintf(
"\r\n--%s\r\nContent-Type: %s\r\nContent-Range: %s\r\n\r\n",
self::BOUNDARY,
$mimeType,
$ranges->format($range)
);
$headers[$i]= $header;
$length+= strlen($header) + $range->length();
}

$response->header('Content-Type', 'multipart/byteranges; boundary='.self::BOUNDARY);
$response->header('Content-Length', $length);
$response->flush();

$out= $response->stream($length);
foreach ($ranges->sets() as $i => $range) {
$output->write($headers[$i]);
$file->seek($range->start());
$this->copy($output, $file, $range->length());
$out->write($headers[$i]);
yield from $this->copy($out, $file, $range);
}
$output->write($trailer);
$out->write($trailer);
}
} finally {
$file->close();
$output->close();
$out->close();
}
}
}
129 changes: 44 additions & 85 deletions src/test/php/web/unittest/handler/FilesFromTest.class.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,24 @@ private function assertResponse($expected, $response) {
));
}

/**
* Returns
*
* @param web.handler.FilesFrom $files
* @param web.Request $req
* @return web.Response
*/
private function handle($files, $req) {
$res= new Response(new TestOutput());

try {
foreach ($files->handle($req, $res) ?? [] as $_) { }
return $res;
} finally {
$res->end();
}
}

/** @return void */
public function tearDown() {
foreach ($this->cleanup as $folder) {
Expand All @@ -70,12 +88,7 @@ public function can_create() {

#[Test]
public function existing_file() {
$req= new Request(new TestInput('GET', '/test.html'));
$res= new Response(new TestOutput());

$files= (new FilesFrom($this->pathWith(['test.html' => 'Test'])));
$files->handle($req, $res);

$files= new FilesFrom($this->pathWith(['test.html' => 'Test']));
$this->assertResponse(
"HTTP/1.1 200 OK\r\n".
"Accept-Ranges: bytes\r\n".
Expand All @@ -85,33 +98,25 @@ public function existing_file() {
"Content-Length: 4\r\n".
"\r\n".
"Test",
$res
$this->handle($files, new Request(new TestInput('GET', '/test.html')))
);
}

#[Test]
public function existing_file_unmodified_since() {
$req= new Request(new TestInput('GET', '/test.html', ['If-Modified-Since' => gmdate('D, d M Y H:i:s T', time() + 1)]));
$res= new Response(new TestOutput());

$files= (new FilesFrom($this->pathWith(['test.html' => 'Test'])));
$files->handle($req, $res);

$files= new FilesFrom($this->pathWith(['test.html' => 'Test']));
$this->assertResponse(
"HTTP/1.1 304 Not Modified\r\n".
"\r\n",
$res
$this->handle($files, new Request(new TestInput('GET', '/test.html', [
'If-Modified-Since' => gmdate('D, d M Y H:i:s T', time() + 1)
])))
);
}

#[Test]
public function index_html() {
$req= new Request(new TestInput('GET', '/'));
$res= new Response(new TestOutput());

$files= (new FilesFrom($this->pathWith(['index.html' => 'Home'])));
$files->handle($req, $res);

$files= new FilesFrom($this->pathWith(['index.html' => 'Home']));
$this->assertResponse(
"HTTP/1.1 200 OK\r\n".
"Accept-Ranges: bytes\r\n".
Expand All @@ -121,89 +126,63 @@ public function index_html() {
"Content-Length: 4\r\n".
"\r\n".
"Home",
$res
$this->handle($files, new Request(new TestInput('GET', '/')))
);
}

#[Test]
public function redirect_if_trailing_slash_missing() {
$req= new Request(new TestInput('GET', '/preview'));
$res= new Response(new TestOutput());

$files= (new FilesFrom($this->pathWith(['preview' => ['index.html' => 'Home']])));
$files->handle($req, $res);

$files= new FilesFrom($this->pathWith(['preview' => ['index.html' => 'Home']]));
$this->assertResponse(
"HTTP/1.1 301 Moved Permanently\r\n".
"Location: preview/\r\n".
"\r\n",
$res
$this->handle($files, new Request(new TestInput('GET', '/preview')))
);
}

#[Test]
public function non_existant_file() {
$req= new Request(new TestInput('GET', '/test.html'));
$res= new Response(new TestOutput());

$files= (new FilesFrom($this->pathWith([])));
$files->handle($req, $res);

$files= new FilesFrom($this->pathWith([]));
$this->assertResponse(
"HTTP/1.1 404 Not Found\r\n".
"Content-Type: text/plain\r\n".
"Content-Length: 35\r\n".
"\r\n".
"The file '/test.html' was not found",
$res
$this->handle($files, new Request(new TestInput('GET', '/test.html')))
);
}

#[Test]
public function non_existant_index_html() {
$req= new Request(new TestInput('GET', '/'));
$res= new Response(new TestOutput());

$files= (new FilesFrom($this->pathWith([])));
$files->handle($req, $res);

$files= new FilesFrom($this->pathWith([]));
$this->assertResponse(
"HTTP/1.1 404 Not Found\r\n".
"Content-Type: text/plain\r\n".
"Content-Length: 26\r\n".
"\r\n".
"The file '/' was not found",
$res
$this->handle($files, new Request(new TestInput('GET', '/')))
);
}

#[Test, Values(['/../credentials', '/static/../../credentials'])]
public function cannot_access_below_path_root($uri) {
$req= new Request(new TestInput('GET', $uri));
$res= new Response(new TestOutput());

$path= $this->pathWith(['credentials' => 'secret']);
$files= new FilesFrom(new Folder($path, 'webroot'));
$files->handle($req, $res);

$files= new FilesFrom(new Folder($this->pathWith(['credentials' => 'secret']), 'webroot'));
$this->assertResponse(
"HTTP/1.1 404 Not Found\r\n".
"Content-Type: text/plain\r\n".
"Content-Length: 37\r\n".
"\r\n".
"The file '/credentials' was not found",
$res
$this->handle($files, new Request(new TestInput('GET', $uri)))
);
}

#[Test, Values([['0-3', 'Home'], ['4-7', 'page'], ['0-0', 'H'], ['4-4', 'p'], ['7-7', 'e']])]
public function range_with_start_and_end($range, $result) {
$req= new Request(new TestInput('GET', '/', ['Range' => 'bytes='.$range]));
$res= new Response(new TestOutput());

$files= (new FilesFrom($this->pathWith(['index.html' => 'Homepage'])));
$files->handle($req, $res);

$files= new FilesFrom($this->pathWith(['index.html' => 'Homepage']));
$this->assertResponse(
"HTTP/1.1 206 Partial Content\r\n".
"Accept-Ranges: bytes\r\n".
Expand All @@ -214,18 +193,13 @@ public function range_with_start_and_end($range, $result) {
"Content-Length: ".strlen($result)."\r\n".
"\r\n".
$result,
$res
$this->handle($files, new Request(new TestInput('GET', '/', ['Range' => 'bytes='.$range])))
);
}

#[Test]
public function range_from_offset_until_end() {
$req= new Request(new TestInput('GET', '/', ['Range' => 'bytes=4-']));
$res= new Response(new TestOutput());

$files= (new FilesFrom($this->pathWith(['index.html' => 'Homepage'])));
$files->handle($req, $res);

$files= new FilesFrom($this->pathWith(['index.html' => 'Homepage']));
$this->assertResponse(
"HTTP/1.1 206 Partial Content\r\n".
"Accept-Ranges: bytes\r\n".
Expand All @@ -236,18 +210,13 @@ public function range_from_offset_until_end() {
"Content-Length: 4\r\n".
"\r\n".
"page",
$res
$this->handle($files, new Request(new TestInput('GET', '/', ['Range' => 'bytes=4-'])))
);
}

#[Test, Values([0, 8192, 10000])]
public function range_last_four_bytes($offset) {
$req= new Request(new TestInput('GET', '/', ['Range' => 'bytes=-4']));
$res= new Response(new TestOutput());

$files= (new FilesFrom($this->pathWith(['index.html' => str_repeat('*', $offset).'Homepage'])));
$files->handle($req, $res);

$files= new FilesFrom($this->pathWith(['index.html' => str_repeat('*', $offset).'Homepage']));
$this->assertResponse(
"HTTP/1.1 206 Partial Content\r\n".
"Accept-Ranges: bytes\r\n".
Expand All @@ -258,37 +227,27 @@ public function range_last_four_bytes($offset) {
"Content-Length: 4\r\n".
"\r\n".
"page",
$res
$this->handle($files, new Request(new TestInput('GET', '/', ['Range' => 'bytes=-4'])))
);
}

#[Test, Values(['bytes=0-2000', 'bytes=4-2000', 'bytes=2000-', 'bytes=2000-2001', 'bytes=2000-0', 'bytes=4-0', 'characters=0-'])]
public function range_unsatisfiable($range) {
$req= new Request(new TestInput('GET', '/', ['Range' => $range]));
$res= new Response(new TestOutput());

$files= (new FilesFrom($this->pathWith(['index.html' => 'Homepage'])));
$files->handle($req, $res);

$files= new FilesFrom($this->pathWith(['index.html' => 'Homepage']));
$this->assertResponse(
"HTTP/1.1 416 Range Not Satisfiable\r\n".
"Accept-Ranges: bytes\r\n".
"Last-Modified: <Date>\r\n".
"X-Content-Type-Options: nosniff\r\n".
"Content-Range: bytes */8\r\n".
"\r\n",
$res
$this->handle($files, new Request(new TestInput('GET', '/', ['Range' => $range])))
);
}

#[Test]
public function multi_range() {
$req= new Request(new TestInput('GET', '/', ['Range' => 'bytes=0-3,4-7']));
$res= new Response(new TestOutput());

$files= (new FilesFrom($this->pathWith(['index.html' => 'Homepage'])));
$files->handle($req, $res);

$files= new FilesFrom($this->pathWith(['index.html' => 'Homepage']));
$this->assertResponse(
"HTTP/1.1 206 Partial Content\r\n".
"Accept-Ranges: bytes\r\n".
Expand All @@ -306,7 +265,7 @@ public function multi_range() {
"Content-Range: bytes 4-7/8\r\n\r\n".
"page".
"\r\n--594fa07300f865fe--\r\n",
$res
$this->handle($files, new Request(new TestInput('GET', '/', ['Range' => 'bytes=0-3,4-7'])))
);
}
}