From aa0251c3547b069297cab83eca13fc3da5d71296 Mon Sep 17 00:00:00 2001 From: korelstar Date: Fri, 20 Jan 2017 13:24:52 +0100 Subject: [PATCH 1/4] 3rd-party API: ETag for faster synchronization --- controller/notesapicontroller.php | 16 +++++++++++++++- db/note.php | 7 +++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/controller/notesapicontroller.php b/controller/notesapicontroller.php index 4f55b3f93..f1044558a 100644 --- a/controller/notesapicontroller.php +++ b/controller/notesapicontroller.php @@ -64,6 +64,17 @@ private function excludeFields(Note $note, array $exclude) { } + private function excludeIfNotModified(Note $note, array $etags) { + $etag = $note->getId().'-'.$note->getEtag(); + if(in_array($etag, $etags)) { + $vars = get_object_vars($note); + unset($vars['id']); + $this->excludeFields($note, array_keys($vars)); + } + return $note; + } + + /** * @NoAdminRequired * @CORS @@ -74,11 +85,14 @@ private function excludeFields(Note $note, array $exclude) { */ public function index($exclude='') { $exclude = explode(',', $exclude); + $etags = array_key_exists('HTTP_X_NOTES_ETAGS', $_SERVER) ? $_SERVER['HTTP_X_NOTES_ETAGS'] : ''; + $etags = explode(',', $etags); $notes = $this->service->getAll($this->userId); foreach ($notes as $note) { $note = $this->excludeFields($note, $exclude); + $note = $this->excludeIfNotModified($note, $etags); } - return new DataResponse($notes); + return (new DataResponse($notes))->addHeader('Vary', 'X-Notes-Etags'); } diff --git a/db/note.php b/db/note.php index f277ea061..4e83c2aee 100644 --- a/db/note.php +++ b/db/note.php @@ -19,6 +19,8 @@ * Class Note * @method integer getId() * @method void setId(integer $value) + * @method string getEtag() + * @method void setEtag(string $value) * @method integer getModified() * @method void setModified(integer $value) * @method string getTitle() @@ -33,6 +35,7 @@ */ class Note extends Entity { + public $etag; public $modified; public $title; public $category; @@ -60,8 +63,12 @@ public static function fromFile(File $file, Folder $notesFolder, $tags=[]){ $note->setFavorite(true); //unset($tags[array_search(\OC\Tags::TAG_FAVORITE, $tags)]); } + $note->updateETag(); $note->resetUpdatedFields(); return $note; } + private function updateETag() { + $this->setEtag(md5($this->title.$this->content.$this->modified.$this->favorite.$this->category)); + } } From d585c82c914025bb9fb8961b911a49519bffb0de Mon Sep 17 00:00:00 2001 From: korelstar Date: Fri, 20 Jan 2017 14:26:15 +0100 Subject: [PATCH 2/4] adjust tests --- tests/unit/controller/NotesApiControllerTest.php | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/unit/controller/NotesApiControllerTest.php b/tests/unit/controller/NotesApiControllerTest.php index 3ce8cab6d..3f3feeab9 100644 --- a/tests/unit/controller/NotesApiControllerTest.php +++ b/tests/unit/controller/NotesApiControllerTest.php @@ -87,12 +87,14 @@ public function testGetAllHide(){ $this->assertEquals(json_encode([ [ + 'etag' => null, 'modified' => 123, 'category' => null, 'favorite' => false, 'id' => 3, ], [ + 'etag' => null, 'modified' => 111, 'category' => null, 'favorite' => false, @@ -139,6 +141,7 @@ public function testGetHide(){ $response = $this->controller->get(3, 'title,content'); $this->assertEquals(json_encode([ + 'etag' => null, 'modified' => 123, 'category' => null, 'favorite' => false, From 7f99a55a9f9077d8f77d1e0a63375e94231e6745 Mon Sep 17 00:00:00 2001 From: korelstar Date: Tue, 24 Jan 2017 16:39:28 +0100 Subject: [PATCH 3/4] minimize size of ETags and Header --- controller/notesapicontroller.php | 4 ++-- db/note.php | 24 +++++++++++++++++++++++- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/controller/notesapicontroller.php b/controller/notesapicontroller.php index f1044558a..566b6e652 100644 --- a/controller/notesapicontroller.php +++ b/controller/notesapicontroller.php @@ -65,7 +65,7 @@ private function excludeFields(Note $note, array $exclude) { private function excludeIfNotModified(Note $note, array $etags) { - $etag = $note->getId().'-'.$note->getEtag(); + $etag = $note->getEtag(); if(in_array($etag, $etags)) { $vars = get_object_vars($note); unset($vars['id']); @@ -86,7 +86,7 @@ private function excludeIfNotModified(Note $note, array $etags) { public function index($exclude='') { $exclude = explode(',', $exclude); $etags = array_key_exists('HTTP_X_NOTES_ETAGS', $_SERVER) ? $_SERVER['HTTP_X_NOTES_ETAGS'] : ''; - $etags = explode(',', $etags); + $etags = str_split($etags, 20); $notes = $this->service->getAll($this->userId); foreach ($notes as $note) { $note = $this->excludeFields($note, $exclude); diff --git a/db/note.php b/db/note.php index 4e83c2aee..a86438e0c 100644 --- a/db/note.php +++ b/db/note.php @@ -69,6 +69,28 @@ public static function fromFile(File $file, Folder $notesFolder, $tags=[]){ } private function updateETag() { - $this->setEtag(md5($this->title.$this->content.$this->modified.$this->favorite.$this->category)); + $etag = ''; + // collect all relevant attributes + $data = ''; + foreach(get_object_vars($this) as $key => $val) { + if($key!='etag') { + $data .= $val; + } + } + // create binary checksum + $md5 = md5($data, true); + // binary-to-text using a base85 derivate: + // - only 25% larger than binary data + // - replace problematic characters by not-used ones (no escaping necessary in JSON or HTTP-header) + // - the result has always the same length (20 characters for 16byte md5-checksum) + foreach(unpack('N*', $md5) as $chunk) { + for ($a = 0; $a < 5; $a++) { + $b = intval($chunk / (pow(85,4 - $a))); + $chr = str_replace(array('\\', '/', '<', '>'), array('z', '|', '{', '}'), chr($b + 35)); + $etag .= $chr; + $chunk -= $b * pow(85,4 - $a); + } + } + $this->setEtag($etag); } } From 2731e5a469f898ed17f605660d5e23c43b045cc8 Mon Sep 17 00:00:00 2001 From: korelstar Date: Tue, 24 Jan 2017 16:47:42 +0100 Subject: [PATCH 4/4] fix discouraged and null-etag --- controller/notesapicontroller.php | 2 +- db/note.php | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/controller/notesapicontroller.php b/controller/notesapicontroller.php index 566b6e652..52cd8c464 100644 --- a/controller/notesapicontroller.php +++ b/controller/notesapicontroller.php @@ -66,7 +66,7 @@ private function excludeFields(Note $note, array $exclude) { private function excludeIfNotModified(Note $note, array $etags) { $etag = $note->getEtag(); - if(in_array($etag, $etags)) { + if($etag!==null && in_array($etag, $etags)) { $vars = get_object_vars($note); unset($vars['id']); $this->excludeFields($note, array_keys($vars)); diff --git a/db/note.php b/db/note.php index a86438e0c..f93c67cf6 100644 --- a/db/note.php +++ b/db/note.php @@ -73,7 +73,7 @@ private function updateETag() { // collect all relevant attributes $data = ''; foreach(get_object_vars($this) as $key => $val) { - if($key!='etag') { + if($key!=='etag') { $data .= $val; } }