[CF 5874] v2 - Fix decompression bug in astreamer_lz4

This branch was automatically generated by a robot using patches from an
email thread registered at:

https://commitfest.postgresql.org/patch/5874

The branch will be overwritten each time a new patch version is posted to
the thread, and also periodically to check for bitrot caused by changes
on the master branch.

Patch(es): https://www.postgresql.org/message-id/CAMEv5_u0BWA+tcczoKYdBXMtkwHYfgZLoe0GsH_8pqNM6bc2iQ@mail.gmail.com
Author(s): Mikhail Gribkov

Author: Commitfest Bot

src/bin/pg_verifybackup/t/008_untar.pl

@@ -16,6 +16,24 @@
 $primary->init(allows_streaming => 1);
 $primary->start;
 
+# Create randomly filled file at the root of the data folder to check possible
+# decompression issues. There was a bug reported, that didn't show up with
+# a regular data folder contents. The bug itself was only detected for lz4
+# decompression with levels 0 and 1, and a minimal file size about 512k. It
+# doesn't seem worthy checking too wide variety of combinations, but at least
+# we'll add this file for all compression methods checks.
+# (https://www.postgresql.org/message-id/CAMEv5_uQS1Hg6KCaEP2JkrTBbZ-nXQhxomWrhYQvbdzR-zy-wA%40mail.gmail.com)
+my $junk_data = $primary->safe_psql(
+	'postgres', qq(
+		SELECT string_agg(encode(sha256(i::bytea), 'hex'), '')
+		FROM generate_series(1, 10240) s(i);));
+my $data_dir = $primary->data_dir;
+my $junk_file = "$data_dir/junk";
+open my $jf, '>', $junk_file
+	or die "Could not create junk file: $!";
+print $jf $junk_data;
+close $jf;
+
 # Create a tablespace directory.
 my $source_ts_path = PostgreSQL::Test::Utils::tempdir_short();
 
@@ -52,6 +70,12 @@
 		'backup_archive' => [ 'base.tar.lz4', "$tsoid.tar.lz4" ],
 		'enabled' => check_pg_config("#define USE_LZ4 1")
 	},
+	{
+		'compression_method' => 'lz4',
+		'backup_flags' => [ '--compress', 'server-lz4:5' ],
+		'backup_archive' => [ 'base.tar.lz4', "$tsoid.tar.lz4" ],
+		'enabled' => check_pg_config("#define USE_LZ4 1")
+	},
 	{
 		'compression_method' => 'zstd',
 		'backup_flags' => [ '--compress', 'server-zstd' ],
@@ -111,5 +135,4 @@
 		rmtree($extract_path);
 	}
 }
-
 done_testing();

src/bin/pg_verifybackup/t/010_client_untar.pl

@@ -15,6 +15,24 @@
 $primary->init(allows_streaming => 1);
 $primary->start;
 
+# Create randomly filled file at the root of the data folder to check possible
+# decompression issues. There was a bug reported, that didn't show up with
+# a regular data folder contents. The bug itself was only detected for lz4
+# decompression with levels 0 and 1, and a minimal file size about 512k. It
+# doesn't seem worthy checking too wide variety of combinations, but at least
+# we'll add this file for all compression methods checks.
+# (https://www.postgresql.org/message-id/CAMEv5_uQS1Hg6KCaEP2JkrTBbZ-nXQhxomWrhYQvbdzR-zy-wA%40mail.gmail.com)
+my $junk_data = $primary->safe_psql(
+	'postgres', qq(
+		SELECT string_agg(encode(sha256(i::bytea), 'hex'), '')
+		FROM generate_series(1, 10240) s(i);));
+my $data_dir = $primary->data_dir;
+my $junk_file = "$data_dir/junk";
+open my $jf, '>', $junk_file
+	or die "Could not create junk file: $!";
+print $jf $junk_data;
+close $jf;
+
 my $backup_path = $primary->backup_dir . '/client-backup';
 my $extract_path = $primary->backup_dir . '/extracted-backup';
 
@@ -37,6 +55,12 @@
 		'backup_archive' => 'base.tar.lz4',
 		'enabled' => check_pg_config("#define USE_LZ4 1")
 	},
+	{
+		'compression_method' => 'lz4',
+		'backup_flags' => [ '--compress', 'client-lz4:1' ],
+		'backup_archive' => 'base.tar.lz4',
+		'enabled' => check_pg_config("#define USE_LZ4 1")
+	},
 	{
 		'compression_method' => 'zstd',
 		'backup_flags' => [ '--compress', 'client-zstd:5' ],
@@ -125,5 +149,4 @@
 		rmtree($backup_path);
 	}
 }
-
 done_testing();

src/fe_utils/astreamer_lz4.c

@@ -322,9 +322,9 @@ astreamer_lz4_decompressor_content(astreamer *streamer,
 
 	mystreamer = (astreamer_lz4_frame *) streamer;
 	next_in = (uint8 *) data;
-	next_out = (uint8 *) mystreamer->base.bbs_buffer.data;
+	next_out = (uint8 *) mystreamer->base.bbs_buffer.data + mystreamer->bytes_written;
 	avail_in = len;
-	avail_out = mystreamer->base.bbs_buffer.maxlen;
+	avail_out = mystreamer->base.bbs_buffer.maxlen - mystreamer->bytes_written;
 
 	while (avail_in > 0)
 	{