liblzma: Fix lzma_memlimit_set(strm, 0).
The 0 got treated specially in a buggy way and as a result
the function did nothing. The API doc said that 0 was supposed
to return LZMA_PROG_ERROR but it didn't.
Now 0 is treated as if 1 had been specified. This is done because
0 is already used to indicate an error from lzma_memlimit_get()
and lzma_memusage().
In addition, lzma_memlimit_set() no longer checks that the new
limit is at least LZMA_MEMUSAGE_BASE. It's counter-productive
for the Index decoder and was actually needed only by the
auto decoder. Auto decoder has now been modified to check for
LZMA_MEMUSAGE_BASE.
diff --git a/src/liblzma/api/lzma/base.h b/src/liblzma/api/lzma/base.h
index 7a31a42..a6005ac 100644
--- a/src/liblzma/api/lzma/base.h
+++ b/src/liblzma/api/lzma/base.h
@@ -644,11 +644,16 @@
* This function is supported only when *strm has been initialized with
* a function that takes a memlimit argument.
*
+ * liblzma 5.2.3 and earlier has a bug where memlimit value of 0 causes
+ * this function to do nothing (leaving the limit unchanged) and still
+ * return LZMA_OK. Later versions treat 0 as if 1 had been specified (so
+ * lzma_memlimit_get() will return 1 even if you specify 0 here).
+ *
* \return - LZMA_OK: New memory usage limit successfully set.
* - LZMA_MEMLIMIT_ERROR: The new limit is too small.
* The limit was not changed.
* - LZMA_PROG_ERROR: Invalid arguments, e.g. *strm doesn't
- * support memory usage limit or memlimit was zero.
+ * support memory usage limit.
*/
extern LZMA_API(lzma_ret) lzma_memlimit_set(
lzma_stream *strm, uint64_t memlimit) lzma_nothrow;
diff --git a/src/liblzma/common/auto_decoder.c b/src/liblzma/common/auto_decoder.c
index 479b150..6895c7c 100644
--- a/src/liblzma/common/auto_decoder.c
+++ b/src/liblzma/common/auto_decoder.c
@@ -139,7 +139,10 @@
// the current memory usage.
*memusage = LZMA_MEMUSAGE_BASE;
*old_memlimit = coder->memlimit;
+
ret = LZMA_OK;
+ if (new_memlimit != 0 && new_memlimit < *memusage)
+ ret = LZMA_MEMLIMIT_ERROR;
}
if (ret == LZMA_OK && new_memlimit != 0)
diff --git a/src/liblzma/common/common.c b/src/liblzma/common/common.c
index 28aa2b7..57e3f8e 100644
--- a/src/liblzma/common/common.c
+++ b/src/liblzma/common/common.c
@@ -435,8 +435,10 @@
|| strm->internal->next.memconfig == NULL)
return LZMA_PROG_ERROR;
- if (new_memlimit != 0 && new_memlimit < LZMA_MEMUSAGE_BASE)
- return LZMA_MEMLIMIT_ERROR;
+ // Zero is a special value that cannot be used as an actual limit.
+ // If 0 was specified, use 1 instead.
+ if (new_memlimit == 0)
+ new_memlimit = 1;
return strm->internal->next.memconfig(strm->internal->next.coder,
&memusage, &old_memlimit, new_memlimit);
