blob: 54617d9ef92e752ece055b88e46dbb06dfc1601e [file] [log] [blame]
From e284cfe27457239e932038fb90084c91f4229c36 Mon Sep 17 00:00:00 2001
From: Jonathan Nieder <>
Date: Sat, 16 Jun 2012 05:57:42 -0500
Subject: liblzma: make dlopen()-based liblzma2 compatibility optional
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Suppose I want to build a statically linked program:
gcc -static -o app app.c -lrpm -llzma
Suppose further that librpm.a was built against a pre-5.0 version of
liblzma so it does not allocate as much space for reserved fields at
the end of lzma_stream as the current API requires.
(This is a hypothetical scenario --- Debian librpm does not provide a
static library.)
If liblzma uses unpatched lzma_code() from XZ Utils >= 5.0, then
during calls to librpm that try to compress or decompress an
xz-compressed RPM, lzma_codes reserved field checks will overflow the
buffer and segfault.
If liblzma uses the modified version of lzma_code() which asks libdl
if is resident and refrains from checking reserved fields
past the end of the old lzma_stream struct when the answer is "yes",
the behavior is no better. The dynamic library is _not_
resident, so lzma_code() dutifully reads reserved fields past the end
of the buffer --- segfault.
So the only safe behavior in the static library is to unconditionally
disable checks that might break for callers we want to continue to
The new "./configure --enable-liblzma2-compat" option implements all
three sets of semantics:
- "./configure --disable-liblzma2-compat" means to check the full set
of reserved fields unconditionally. You can use this to check how
your application would behave with the unpatched library.
- "./configure --enable-liblzma2-compat=auto" means to skip checks of
reserved fields past the old end of struct lzma_stream when is resident. If a DSO built against liblzma2 shares
the process image, the ABI-incompatible checks are skipped for
safety, whereas in the usual case when no such DSO is resident, the
full set of checks is run to help application developers remember
to zero all reserved fields.
- "./configure --enable-liblzma2-compat" makes liblzma skip the
ABI-incompatible checks unconditionallty. You can use this if you
want your copy of liblzma to be usable by static libraries that
were built against the old library.
Patch-Name: liblzma-make-dlopen-based-liblzma2-compatibility-opt.patch
Signed-off-by: Jonathan Nieder <>
--- | 33 +++++++++++++++++++++++++++++++--
src/liblzma/common/common.c | 40 +++++++++++++++++++++++++++++++++++-----
src/liblzma/common/common.h | 2 ++
3 files changed, 68 insertions(+), 7 deletions(-)
diff --git a/ b/
index 6dae0b9756d6..d17629e0e7f6 100644
--- a/
+++ b/
@@ -548,10 +548,39 @@ case $enable_threads in
AM_CONDITIONAL([COND_THREADS], [test "x$enable_threads" != xno])
-# As a Debian-specific hack, liblzma uses dlopen() to check if extra
+# As a Debian-specific hack, liblzma can use dlopen() to check if extra
# paranoia is needed because unversioned symbols from are
# present in the same process. See src/liblzma/common/common.c.
-AC_SEARCH_LIBS([dlopen], [dl])
+AC_MSG_CHECKING([if lzma_code checks should be relaxed for compatibility])
+AC_ARG_ENABLE([liblzma2-compat], [AC_HELP_STRING([--enable-liblzma2-compat],
+ [Relax run-time checks to accomodate old binaries built
+ with smaller sizeof(lzma_stream). The default is "dynamic",
+ which means to only use the relaxed checks when the dynamic
+ loader reports that is loaded in the same process.])],
+ [], [enable_liblzma2_compat=dynamic])
+case $enable_liblzma2_compat in
+ AC_SEARCH_LIBS([dlopen], [dl])
+ [Define to 1 to use dlopen() to check if lzma_code() checks
+ should be more tolerant because the process is also linked to
+ liblzma from Debian 6.0.])
+ AC_MSG_RESULT([auto])
+ ;;
+ [Define to 1 to unconditionally make lzma_code() checks tolerant
+ to accomodate callers built against liblzma from Debian 6.0.])
+ AC_MSG_RESULT([yes])
+ ;;
+ ;;
+ AC_MSG_ERROR([--enable-liblzma2: unrecognized value $enable_liblzma2_compat])
+ ;;
echo "Initializing Libtool:"
diff --git a/src/liblzma/common/common.c b/src/liblzma/common/common.c
index 5474211cf0ca..79427b005b4b 100644
--- a/src/liblzma/common/common.c
+++ b/src/liblzma/common/common.c
@@ -164,16 +164,46 @@ lzma_next_end(lzma_next_coder *next, const lzma_allocator *allocator)
// External to internal API wrapper //
-static bool
+static void
+init_liblzma2_compat(lzma_stream *strm)
void *handle = dlopen("", RTLD_LAZY | RTLD_NOLOAD);
if (handle) {
- return true;
+ strm->internal->liblzma2_compat = true;
+ return;
+ strm->internal->liblzma2_compat = false;
+static bool
+liblzma2_loaded(lzma_stream *strm)
+ return strm->internal->liblzma2_compat;
+static void
+init_liblzma2_compat(lzma_stream *strm)
+static bool liblzma2_loaded(lzma_stream *strm)
+ return true;
+static bool liblzma2_loaded(lzma_stream *strm)
return false;
extern lzma_ret
lzma_strm_init(lzma_stream *strm)
@@ -188,7 +218,7 @@ lzma_strm_init(lzma_stream *strm)
strm->internal->next = LZMA_NEXT_CODER_INIT;
- strm->internal->liblzma2_compat = liblzma2_loaded();
+ init_liblzma2_compat(strm);
@@ -241,7 +271,7 @@ lzma_code(lzma_stream *strm, lzma_action action)
|| strm->reserved_ptr4 != NULL)
- if (strm->internal->liblzma2_compat)
+ if (liblzma2_loaded(strm))
; /* Enough checks. */
else if (strm->reserved_int1 != 0
|| strm->reserved_int2 != 0
diff --git a/src/liblzma/common/common.h b/src/liblzma/common/common.h
index b056e417a29e..eb1d0525c07d 100644
--- a/src/liblzma/common/common.h
+++ b/src/liblzma/common/common.h
@@ -227,9 +227,11 @@ struct lzma_internal_s {
/// made (no input consumed and no output produced by next.code).
bool allow_buf_error;
/// Indicates whether we are sharing a process image with
/// and need to tread carefully.
bool liblzma2_compat;