Merge pull request facebook#983 from terrelln/dict-wlog

Increase windowLog from CDict based on the srcSize when known

Author: Cyan4973

lib/compress/zstd_compress.c

@@ -2498,6 +2498,15 @@ size_t ZSTD_compressBegin_usingCDict_advanced(
     if (cdict==NULL) return ERROR(dictionary_wrong);
     {   ZSTD_CCtx_params params = cctx->requestedParams;
         params.cParams = ZSTD_getCParamsFromCDict(cdict);
+        /* Increase window log to fit the entire dictionary and source if the
+         * source size is known. Limit the increase to 19, which is the
+         * window log for compression level 1 with the largest source size.
+         */
+        if (pledgedSrcSize != ZSTD_CONTENTSIZE_UNKNOWN) {
+            U32 const limitedSrcSize = (U32)MIN(pledgedSrcSize, 1U << 19);
+            U32 const limitedSrcLog = limitedSrcSize > 1 ? ZSTD_highbit32(limitedSrcSize - 1) + 1 : 1;
+            params.cParams.windowLog = MAX(params.cParams.windowLog, limitedSrcLog);
+        }
         params.fParams = fParams;
         return ZSTD_compressBegin_internal(cctx,
                                            NULL, 0, ZSTD_dm_auto,

tests/fuzzer.c

@@ -1205,6 +1205,28 @@ static int basicUnitTests(U32 seed, double compressibility)
     if (strcmp("No error detected", ZSTD_getErrorName(ZSTD_error_GENERIC)) != 0) goto _output_error;
     DISPLAYLEVEL(3, "OK \n");
 
+    DISPLAYLEVEL(4, "test%3i : testing ZSTD dictionary sizes : ", testNb++);
+    RDG_genBuffer(CNBuffer, CNBuffSize, compressibility, 0., seed);
+    {
+        size_t const size = MIN(128 KB, CNBuffSize);
+        ZSTD_CCtx* const cctx = ZSTD_createCCtx();
+        ZSTD_CDict* const lgCDict = ZSTD_createCDict(CNBuffer, size, 1);
+        ZSTD_CDict* const smCDict = ZSTD_createCDict(CNBuffer, 1 KB, 1);
+        ZSTD_frameHeader lgHeader;
+        ZSTD_frameHeader smHeader;
+
+        CHECK_Z(ZSTD_compress_usingCDict(cctx, compressedBuffer, compressedBufferSize, CNBuffer, size, lgCDict));
+        CHECK_Z(ZSTD_getFrameHeader(&lgHeader, compressedBuffer, compressedBufferSize));
+        CHECK_Z(ZSTD_compress_usingCDict(cctx, compressedBuffer, compressedBufferSize, CNBuffer, size, smCDict));
+        CHECK_Z(ZSTD_getFrameHeader(&smHeader, compressedBuffer, compressedBufferSize));
+
+        if (lgHeader.windowSize != smHeader.windowSize) goto _output_error;
+
+        ZSTD_freeCDict(smCDict);
+        ZSTD_freeCDict(lgCDict);
+        ZSTD_freeCCtx(cctx);
+    }
+
 _end:
     free(CNBuffer);
     free(compressedBuffer);