Implement non-blocking API (VirusTotal#1428)

With this change the yr_scanner_scan_mem_blocks function can return ERROR_BLOCK_NOT_READY when the YR_MEMORY_BLOCK_ITERATOR that provides memory blocks to the function is not ready to return the next block. After this error the caller can retry the call to yr_scanner_scan_mem_blocks multiple times until the function succeeds or fails with some other error code.

Co-authored-by: Simon Hardy-Francis <simonhf@gmail.com>

Author: plusvic

.gitignore

@@ -65,11 +65,15 @@ test-alignment
 test-api
 test-arena
 test-arena-stream
+test-async
 test-atoms
 test-bitmask
 test-elf
 test-exception
-test-rules
+test-rules-pass-1
+test-rules-pass-2
+test-rules-pass-3
+test-rules.yarc
 test-pb
 test-pe
 test-re-split

Makefile.am

@@ -54,28 +54,36 @@ yarac_SOURCES = \
 yarac_LDADD = -Llibyara/.libs -lyara
 yarac_DEPENDENCIES = libyara/.libs/libyara.la
 
-test_alignment_SOURCES = tests/test-alignment.c
-test_arena_SOURCES = tests/test-arena.c
+test_alignment_SOURCES = tests/test-alignment.c tests/util.c
+test_alignment_LDADD = libyara/.libs/libyara.a
+test_arena_SOURCES = tests/test-arena.c tests/util.c
 test_arena_LDADD = libyara/.libs/libyara.a
 test_atoms_SOURCES = tests/test-atoms.c tests/util.c
 test_atoms_LDADD = libyara/.libs/libyara.a
-test_rules_SOURCES = tests/test-rules.c tests/util.c
-test_rules_LDADD = libyara/.libs/libyara.a
+test_rules_pass_1_SOURCES = tests/test-rules-pass-1.c tests/util.c
+test_rules_pass_1_LDADD = libyara/.libs/libyara.a
+test_rules_pass_2_SOURCES = tests/test-rules-pass-2.c tests/util.c
+test_rules_pass_2_LDADD = libyara/.libs/libyara.a
+test_rules_pass_3_SOURCES = tests/test-rules-pass-3.c tests/util.c
+test_rules_pass_3_LDADD = libyara/.libs/libyara.a
 test_pe_SOURCES = tests/test-pe.c tests/util.c
 test_pe_LDADD = libyara/.libs/libyara.a
 test_elf_SOURCES = tests/test-elf.c tests/util.c
 test_elf_LDADD = libyara/.libs/libyara.a
-test_version_SOURCES = tests/test-version.c
-test_api_LDADD = libyara/.libs/libyara.a
+test_version_SOURCES = tests/test-version.c tests/util.c
+test_version_LDADD = libyara/.libs/libyara.a
 test_api_SOURCES = tests/test-api.c tests/util.c
-test_bitmask_SOURCES = tests/test-bitmask.c
+test_api_LDADD = libyara/.libs/libyara.a
+test_bitmask_SOURCES = tests/test-bitmask.c tests/util.c
 test_bitmask_LDADD = libyara/.libs/libyara.a
 test_math_SOURCES = tests/test-math.c tests/util.c
 test_math_LDADD = libyara/.libs/libyara.a
-test_stack_SOURCES = tests/test-stack.c
+test_stack_SOURCES = tests/test-stack.c tests/util.c
 test_stack_LDADD = libyara/.libs/libyara.a
-test_re_split_SOURCES = tests/test-re-split.c
+test_re_split_SOURCES = tests/test-re-split.c tests/util.c
 test_re_split_LDADD = libyara/.libs/libyara.a
+test_async_SOURCES = tests/test-async.c tests/util.c
+test_async_LDADD = libyara/.libs/libyara.a
 
 TESTS = $(check_PROGRAMS)
 TESTS_ENVIRONMENT = TOP_SRCDIR=$(top_srcdir)
@@ -85,14 +93,17 @@ check_PROGRAMS = \
   test-alignment \
   test-atoms \
   test-api \
-  test-rules \
+  test-rules-pass-1 \
+  test-rules-pass-2 \
+  test-rules-pass-3 \
   test-pe \
   test-elf \
   test-version \
   test-bitmask \
   test-math \
   test-stack \
-  test-re-split
+  test-re-split \
+  test-async
 
 if POSIX
 # The -fsanitize=address option makes test-exception fail. Include the test

docs/capi.rst

@@ -891,6 +891,31 @@ Functions
 
   Define a string external variable.
 
+.. c:function:: int yr_scanner_scan_mem_blocks(YR_SCANNER* scanner, YR_MEMORY_BLOCK_ITERATOR* iterator)
+
+  .. versionadded:: 3.8.0
+
+  Scan a serie of memory blocks that are provided by a :c:type:`YR_MEMORY_BLOCK_ITERATOR`.
+  The iterator has a pair of `first` and `next` functions, that must return
+  the first and next blocks respectively. The how the data is split in blocks is
+  up to the iterator implementation.
+
+  Returns one of the following error codes:
+
+    :c:macro:`ERROR_SUCCESS`
+
+    :c:macro:`ERROR_INSUFFICIENT_MEMORY`
+
+    :c:macro:`ERROR_TOO_MANY_SCAN_THREADS`
+
+    :c:macro:`ERROR_SCAN_TIMEOUT`
+
+    :c:macro:`ERROR_CALLBACK_ERROR`
+
+    :c:macro:`ERROR_TOO_MANY_MATCHES`
+
+    :c:macro:`ERROR_BLOCK_NOT_READY`
+
 .. c:function:: int yr_scanner_scan_mem(YR_SCANNER* scanner, const uint8_t* buffer, size_t buffer_size)
 
   .. versionadded:: 3.8.0
@@ -1000,3 +1025,7 @@ Error codes
   your rules contains very short or very common strings like ``01 02`` or
   ``FF FF FF FF``. The limit is defined by ``YR_MAX_STRING_MATCHES`` in
   *./include/yara/limits.h*
+
+.. c:macro:: ERROR_BLOCK_NOT_READY
+
+  Next memory block to scan is not ready; custom iterators may return this.